Schulisches/Zotero-Thesis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
Zotero-Thesis/storage/AE8HHNYJ/2408.html
fzzinchemical 02b00ee108 update
2026-01-22 22:01:07 +01:00

389 lines
96 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html> <html lang=en><!--
Page saved with SingleFile
url: https://arxiv.org/abs/2408.08333
saved date: Sun Jan 11 2026 23:53:14 GMT+0100 (Central European Standard Time)
--><meta charset=utf-8>
<title>[2408.08333] CodeMirage: Hallucinations in Code Generated by Large Language Models</title>
<meta name=viewport content="width=device-width, initial-scale=1">
<meta name=msapplication-TileColor content=#da532c>
<meta name=theme-color content=#ffffff>
<style media=screen>body{margin:0;padding:0;background-color:#fff;color:#000;font-family:"Lucida Grande",helvetica,arial,verdana,sans-serif}a:link,a:visited,a:active{text-decoration:none;font-weight:normal}a:hover{text-decoration:underline}img{border:0}em{font-weight:bold;font-style:normal}.primary-subject{font-weight:bold}#cu-identity{font-family:verdana,arial,helvetica,sans-serif;font-size:63.125%;color:#fff;background-color:#222;width:100%;display:flex;justify-content:space-between}#cu-logo{position:relative;left:10px;top:2px;width:300px;height:49px}#cu-logo a img{width:200px}#support-ack{top:12px;right:0%;margin:0 12px 0 0;padding:8px 0;text-align:right;font-size:120%;font-weight:normal;font-family:"Lucida Grande",helvetica,arial,verdana,sans-serif;color:#fff;display:flex;align-items:center;gap:1em;justify-content:right}#support-ack-url{max-width:400px;float:right;display:block}#support-ack-url a{color:#fff;border:none;text-decoration:underline}#support-ack-url a:hover{color:#1e8bc3}#support-ack .btn-header-donate{background-color:white;border:1px solid #b31b1b;padding:.2em 1em 0 1em;border-radius:.5em;color:#b31b1b;height:1.5em;font-size:1.25em;transition:all .5s ease-in-out}#support-ack .btn-header-donate:hover{background-color:#fb595a;color:black}#header{background-color:#b31b1b;color:#fff;margin:0;padding:10px 0 10px 0;border-bottom:2px solid #ccc;position:relative;overflow:auto}#header .header-breadcrumbs{margin:0;font-size:1em;padding:10px 0 .2em 10px;font-style:normal;float:left;display:inline-flex;align-items:center}#header .header-breadcrumbs span{margin-right:5px;margin-left:5px}#header a,#header a:visited{color:#fff;text-decoration:none}#header a:hover{text-decoration:underline}#header form{margin:0 12px 0 0;padding:0;text-align:right;font-size:.8em;line-height:100%}#header form input,#header form select{margin:0}@media screen and (max-width:768px){.search-block.level-right{clear:both!important}#header .header-breadcrumbs{float:none;text-align:center}}main{flex-grow:1}.flex-wrap-footer{display:flex;min-height:100vh;flex-direction:column}footer ul li{display:flex;align-items:center;font-size:14px}footer ul li a{font-size:13.5px}#footer a:visited{text-decoration:none}#footer a:hover{text-decoration:underline}footer{background-color:hsl(0,0%,95%);color:#000;padding:1em 2em;font-size:0.9rem;-webkit-font-smoothing:antialiased;margin-top:6rem}footer a,footer a:visited{color:#000;text-decoration:none;border-bottom:1px solid transparent;line-height:1.75em}footer a:hover,footer a:active{color:#005e9d;border-bottom:1px dotted #005e9d;text-decoration:none}footer ul{padding:0;margin:0}footer .sorry-app-links .help{font-size:0.75rem;margin-bottom:0;line-height:1.75em}footer .sorry-app-links .help a:hover,footer .sorry-app-links .help a:active{border-bottom:1px dotted #005e9d}footer .sorry-app-links svg.icon{margin-bottom:-2px!important}footer .sorry-app-links .icon.filter-black:hover,footer .sorry-app-links .icon.filter-black:active,footer .sorry-app-links a:hover .icon.filter-black,footer .sorry-app-links a:hover .icon.filter-black{fill:#005e9d!important}footer .sorry-app-links .a11y-main-link{font-size:110%;border-bottom:1px solid transparent!important;padding:0;margin:0}@media screen and (max-width:768px){footer .sorry-app-links.column{padding:0}}.endorsers .help{position:relative;top:4rem}@media screen and (min-width:990px){.endorsers .help{top:5rem}}@media screen and (min-width:769px){.columns{display:flex;flex-direction:row}}.icon{width:.9rem;margin-right:.45em;margin-top:-.15rem}.help{font-family:"Lucida Grande","Helvetica Neue",Helvetica,Arial,sans-serif;display:block;font-size:0.75rem;margin-top:0.25rem}#content,#content-inner{margin:.7em;font-size:90%}.button-fancy:hover{border-bottom:1px solid #f3fbff;color:black}.button-fancy span:after{content:"»";position:relative;opacity:0;top:1px;right:-5px;padding-left:10px;transition:0.5s;font-size:20px;color:#1772a0}.button-fancy:hover span{padding-right:15px}.button-fancy:hover span:after{opacity:1;right:0}.banner-minimal a:hover{background-color:#1c1a17;border-color:#dbc6c6;color:#f9f7f7}.columns .column.banner-minimal.forum a:hover{border-color:transparent;background-color:transparent;color:#fff;text-decoration:none}.tagline-cta:hover{background-color:#d9af0c;text-decoration:none}.tagline-cta:hover p{color:#ffffff}#abs-outer,#abs{margin:-0.7em}#abs-outer .leftcolumn{margin:0 0 1em 0;padding:0px;width:calc(100% - 18em);float:left}#abs-outer .mobile-submission-download{display:none}#abs-outer .extra-services{float:right;margin:0;width:18em}#abs-outer .extra-services span.bib-cite-button{color:rgb(0,0,238);font-weight:bold;padding:.35em;display:block;text-transform:capitalize}#abs-outer .extra-services span.bib-cite-button:hover,#abs-outer .extra-services span.bib-cite-button:focus{text-decoration:underline;cursor:pointer}.bib-modal{position:fixed;z-index:10;left:0;top:0;width:100%;height:100%;overflow:auto;background-color:rgba(0,0,0,0.4)}.bib-modal-close:hover,.bib-modal-close:focus{color:black;text-decoration:none;cursor:pointer}#labstabs{clear:both;margin:1em 1.5em}#labstabs .labstabs{display:flex;flex-wrap:wrap;white-space:normal;justify-content:flex-start}#labstabs .labstabs>label{order:1;display:block;padding:.75em 1.5em;font-size:.85em;color:#4a4a4a;margin-right:0.2rem;cursor:pointer;font-weight:bold;transition:background ease 0.2s;border-radius:6px 6px 0 0;background-color:#eef5f9}#labstabs .labstabs .tab{order:99;flex-grow:1;width:100%;display:none;padding:1rem;border:1px solid #828282;z-index:1;top:-1px;position:relative}#labstabs .labstabs .tab h1{font-size:1.25em;font-weight:normal}#labstabs .labstabs input[type="radio"]{display:none}#labstabs .labstabs input[type="radio"]:checked+label{background:#ffffff;border:1px solid #828282;border-bottom:0;z-index:2;color:black}#labstabs .labstabs input[type="radio"]:checked+label+.tab{display:block}@media (max-width:45em){#labstabs{padding-bottom:1em;margin:1em}#labstabs .labstabs .tab,#labstabs .labstabs label{order:initial;width:100%;margin-right:0;margin-top:0.2rem;border-radius:0}#labstabs .labstabs>label{font-size:.8em}#labstabs .labstabs .tab{margin-top:-10px!important;border:2px solid #b8b8b8;border-top:0px;background-color:#ffffff}#labstabs .labstabs label{background-color:#eef5f9;color:black;padding:1em .5em}#labstabs .labstabs .labs-display-bib label{background:transparent;color:black;padding:0}#labstabs .labstabs input[type="radio"]:checked+label{background-color:#acd2e8;color:black;border:2px solid #b8b8b8;border-bottom:0px;padding:1em .5em}#labstabs .labstabs input[type="radio"]:hover,#labstabs .labstabs input[type="radio"]:active{background-color:#abb4b9}}#labstabs .toggle{border:1px dotted #c7d3db;padding:.5em;background-color:#eef5f9}#labstabs .toggle .columns.lab-row{align-items:center}#labstabs .toggle .columns.lab-row .column{padding:.5rem .25rem}#labstabs .column.lab-name{flex-basis:auto;flex-grow:0;font-size:.75rem}#labstabs .tab a{display:inline}#labstabs .column.lab-name em{font-size:11px;color:black}#labstabs .column.lab-name em a{line-height:18px;text-decoration:none;padding:0;border:0;font-style:normal;font-size:11px}#labstabs .column.lab-switch{flex-grow:0}#labstabs .toggle .lab-switch label.switch{position:relative;display:inline-block;width:38px;height:20px;padding:0;background-color:transparent;margin:0}#labstabs .toggle .lab-switch .switch input{opacity:0;width:0;height:0}#labstabs .toggle .lab-switch .slider{position:absolute;cursor:pointer;top:0;left:0;right:0;bottom:0;border-radius:24px;background-color:#ccc;-webkit-transition:.4s;transition:.4s}#labstabs .toggle .lab-switch .slider:before{position:absolute;content:"";height:16px;width:16px;left:2px;bottom:2px;border-radius:50%;background-color:white;-webkit-transition:.4s;transition:.4s}#labstabs .toggle .lab-switch input.enabled+.slider:before{-webkit-transform:translateX(18px);-ms-transform:translateX(86px);transform:translateX(18px)}#abs-outer .subheader{background-color:#eee;color:#000;padding:.25em 0;border-bottom:1px solid #ccc}#abs-outer .subheader h1{margin:0;font-size:1.1em;padding:0 0 .2em 20px;font-weight:bold;font-style:normal}#abs .descriptor{display:none}#abs-outer .extra-services .descriptor{display:none}#abs h1.title{margin:.5em 0 .5em 20px;font-size:x-large;font-weight:bold;line-height:120%}#abs .authors{margin:.5em 0 .5em 20px;font-size:medium;line-height:150%}#abs .authors a{font-size:medium}#abs .dateline{margin:.5em 0 .5em 20px;font-style:italic;font-size:small}#abs blockquote.abstract{line-height:1.55;font-size:1.05em;margin-bottom:1.5em}#abs .metatable{font-size:0.92em;line-height:1.5;margin:0 0 1.5em 20px}#abs .tablecell{padding:.1em .5em 0em 0em;vertical-align:top}#abs .arxivid a{font-weight:normal}#abs-outer .submission-history{margin:1.5em 0 1.5em 20px;font-size:90%;line-height:1.5em}#abs-outer .submission-history h2{font-size:120%;margin:0 0 .25em 0;font-weight:bold}#abs-outer .endorsers{margin:1em 0 1.5em 20px;font-size:small;font-style:italic;clear:both}#abs-outer .header-breadcrumbs-mobile{display:none}#abs-outer #abs .dateline{margin-top:15px;margin-bottom:0}#abs-outer #abs h1.title{margin-top:.25em}.endorsers .help{display:inline-block}@media screen and (max-width:768px){#abs-outer .header-breadcrumbs-mobile{display:block;color:black;font-size:.85em;margin:.25em 0 .5em 1em}#abs-outer #abs a.mobile-submission-download{display:flex;justify-content:center;margin:.7em .25em;border-radius:10px;background-color:#408bd0;padding:.25em 1em;font-weight:800;color:white;text-decoration:none;font-size:20px;text-align:center}#abs-outer .extra-services h2{font-size:15px;margin-bottom:.5em}#abs-outer .extra-services h3{font-size:14px}#abs-outer .leftcolumn,#abs-outer .extra-services{width:100%;float:left}#abs-outer .extra-services,#abs-outer .extra-services .full-text,#abs-outer .extra-services .browse,#abs-outer .extra-services .extra-ref-cite{border:0px;font-size:12px}#abs-outer .extra-services{-webkit-box-shadow:inset 0px 8px 15px 0px rgba(173,173,173,1);-moz-box-shadow:inset 0px 8px 15px 0px rgba(173,173,173,1);box-shadow:inset 0px 8px 15px 0px rgba(173,173,173,1);background-color:#E6E6E6;margin:0 0 1em 0;padding:1em 0}#abs-outer .extra-ref-cite ul li{float:left}#abs-outer .extra-services .full-text ul,#abs-outer .extra-services .extra-ref-cite ul{list-style:none;margin:0;padding:0}#abs-outer .extra-services .full-text ul li,#abs-outer .extra-services .extra-ref-cite ul li{display:inline-block;margin:0 0 .25em 0;padding:0}#abs-outer .extra-services .bookmarks{margin:1em 0 0 0;border-left:0;padding:.25em .5em 0 1em;border-top:2px solid #cccccc;font-size:1em}#abs-outer .extra-services .bookmarks .abs-button-small{margin-top:.25em}#abs-outer .extra-services .browse{margin-top:.5em;border-top:2px solid #cccccc;padding-top:1em}#abs-outer .extra-services .prevnext{margin-top:.5em}#abs-outer .extra-services .browse .current{color:#AB4B02;display:inline}.abs-switch-cat{margin:0 0 1em 0}.browse .abs-switch-cat .switch{display:inline}.browse .abs-switch-cat .switch a{font-weight:bold}.browse .abs-switch-cat .switch .subclass{padding:0;margin-left:-3px}.abs-switch-cat .subclass:before{content:", "}.abs-button{display:inline-block;border-radius:5px;border:1px solid #046BAF;font-size:1.25em;color:#046BAF!important;padding:.5em;background:#E6E6E6;margin-right:.3em}.abs-button-small{font-size:1em;padding:.25em 1em;margin:.75em .5em 0 0}.abs-button-grey{border:1px solid #666666;color:#666666!important}#abs-outer .extra-services span.bib-cite-button{margin:.5em .5em 0 0;display:inline-block!important;border-radius:5px;background:#E6E6E6;border:1px solid #046BAF;font-size:1em!important;padding:.25em 1em;font-weight:normal;text-transform:capitalize}#abs-outer .extra-services .extra-ref-cite ul li{margin:.5em 0;padding:0;height:auto}#abs-outer .extra-services .extra-ref-cite ul li a{margin:.25em .5em .25em 0}#abs-outer .subheader{background-color:#fefefe;padding:.25em 0;border-bottom:1px solid #ccc}#abs-outer .subheader h1{margin:0;font-size:.75em;padding:.2em 0 .2em 1em;font-weight:normal;font-style:normal;color:#b55c06}#abs-outer #abs .dateline{color:#767676;font-size:.85em;font-style:normal;margin:2em 0 0 1em}#abs-outer .submission-history{padding:1em;margin:0;background-color:#f5f5f5;-webkit-box-shadow:inset 0px -6px 15px 0px rgba(219,219,219,1);-moz-box-shadow:inset 0px -6px 15px 0px rgba(219,219,219,1);box-shadow:inset 0px -6px 15px 0px rgba(219,219,219,1)}#abs-outer #abs h1.title{margin:0 .25em 0 .5em;font-size:1.5em}#abs-outer #abs .authors{margin:1em .25em 0 1em;font-size:.9em;line-height:1.5em}#abs-outer #abs .authors a{font-size:inherit}#abs-outer #abs blockquote.abstract{margin:0 1em}#abs-outer #abs .metatable{margin:.75em 0 1.5em 1.5em}#abs-outer #abs a,#abs-outer a,#abs-outer .endorsers a{color:#1777bc}#abs-outer .abs-license a,#abs-outer .submission-history a,#abs-outer .abs-switch-cat .switch a,#abs-outer .extra-services .full-text .abs-license a{color:#046BAF}#abs-outer .endorsers{display:block;float:left;border:1px solid #eee;padding:1em;margin:1em}#abs-outer .endorsers .help{display:none}}.mobile-header{background-color:#b31b1b}.mobile-header .columns{height:65px;align-items:center}.mobile-header .column{height:65px;padding:0 1em;align-items:center}.mobile-header .column:first-child{border-left:0}.mobile-header .column:last-child{border-right:0}.mobile-header .column.logo-cornell{display:flex}.mobile-header .mobile-toggle-block form.mobile-search-form input::-webkit-input-placeholder{color:#94908c}.mobile-header .mobile-toggle-block form.mobile-search-form input::placeholder{color:#94908c}@media screen and (min-width:769px){.mobile-header{display:none}}@media screen and (max-width:500px){.mobile-header .columns{height:80px}.mobile-header .column{height:80px;padding:0 .5em}.mobile-header .column.logo-cornell{justify-content:flex-end;border-left:0!important}}@media screen and (min-width:501px){.mobile-header .column{height:65px}}.extra-services{border-bottom:.35em solid #ddd}.extra-services h3{font-size:medium;font-weight:normal;margin:0 0 0.3em 0;padding-top:0.3em}.full-text{margin:0;padding:.5em 1em .5em 1em;font-size:110%;font-weight:normal;border-bottom:medium solid #ddd;border-left:.35em solid #ddd}.full-text h2{font-size:140%;font-weight:bold;margin:0.1em 0 0 0}.full-text ul{margin:.3em 0 0 1em;padding:0;list-style-type:none}.extra-ref-cite{margin:0;padding:0 1em 0 1em;font-size:90%;border-bottom:medium solid #ddd;border-left:.35em solid #ddd}.extra-ref-cite ul{font-weight:normal;margin:0.3em 0 0 20px;padding-left:0;padding-bottom:0.3em;list-style-type:none}.browse{padding:0 1em 0 1em;font-size:90%;border-bottom:medium solid #ddd;border-left:.35em solid #ddd}.browse .current{padding:0;font-weight:bold}.browse .prevnext{padding:0.2em 0 0 0}.browse .list{padding:0.2em 0 0.5em 0;font-weight:normal}.browse .switch{font-weight:normal;padding:.2em 0em .7em 0em}.browse .switch .subclass{padding-left:1.5em}.bookmarks{clear:both;margin:0;padding:0 1em .5em 1em;font-size:90%;border-left:.35em solid #ddd}.abs-license{font-size:xx-small;padding-top:0.3em}.abs-license .has_license{display:flex;align-items:center;gap:5px}#header #search-arxiv .wrapper-search-arxiv ::-webkit-input-placeholder{color:#222}#header #search-arxiv .wrapper-search-arxiv ::-moz-placeholder{color:#222}#header #search-semantic .wrapper-search-semantic ::-webkit-input-placeholder{color:#264D71}#header #search-semantic .wrapper-search-semantic ::-moz-placeholder{color:#2B6195}#header.cs #search-arxiv .wrapper-search-arxiv ::-webkit-input-placeholder{color:#222}#header.cs #search-arxiv .wrapper-search-arxiv ::-moz-placeholder{color:#222}#header.cs #search-semantic .wrapper-search-semantic ::-webkit-input-placeholder{color:#264D71}#header.cs #search-semantic .wrapper-search-semantic ::-moz-placeholder{color:#2B6195}@media screen and (max-width:768px){#cu-identity{flex-direction:column}#support-ack,#cu-logo{text-align:center;width:100%;left:0px}}@media screen and (max-width:768px){.is-hidden-mobile{display:none!important}.columns.is-mobile{display:flex}}.is-sr-only{border:none!important;clip:rect(0,0,0,0)!important;height:0.01em!important;overflow:hidden!important;padding:0!important;position:absolute!important;white-space:nowrap!important;width:0.01em!important}.button{border-width:1px;cursor:pointer;justify-content:center;padding-bottom:calc(0.5em - 1px);padding-left:1em;padding-right:1em;padding-top:calc(0.5em - 1px);text-align:center;white-space:nowrap}.tabs a:hover{border-bottom-color:#363636;color:#363636}.tabs.is-boxed a:hover{background-color:whitesmoke;border-bottom-color:#dbdbdb}.column{display:block;flex-basis:0;flex-grow:1;flex-shrink:1;padding:0.75rem}.columns.is-vcentered{align-items:center}@media screen and (min-width:769px),print{.columns:not(.is-desktop){display:flex}}@media screen and (min-width:1024px){.columns.is-desktop{display:flex}}.large-data-list .accordion-head:after{content:"˅";padding-left:1em;position:relative;top:.25em;font-size:1.25em;font-weight:100}.large-data-list .accordion-head.open:after{content:"˄";color:#b7b8b9}.trackback-styles .trackback-source a:visited{text-decoration:underline;color:#6c6c6c}.trackback-styles .trackback-source a:hover,.trackback-styles .trackback-source a:active{text-decoration:underline;color:#000000}.bold-divided-list .bold-divided-column ul li:before{color:#000;content:"-";margin:0 6px 0 0;font-weight:bold}svg.icon{height:1em!important}.icon.filter-black{fill:#000000}.filter-dark_grey{fill:#cccccc}a .icon{transition:fill 0.3s ease}a:hover .icon.filter-white{fill:#f5f5f5}a:hover .icon.filter-black,a:hover .icon.filter-grey,a:hover .icon.filter-blue,a:hover .icon.filter-red{fill:#ffffff}</style>
<style media=screen>.search-block.level-right{display:flex;justify-content:flex-end;clear:right}@media screen and (max-width:768px){.search-block.level-right{justify-content:center}.search-block form.level-item{margin-left:12px!important}}.search-block form.level-item,.field.has-addons{display:flex}.search-block p.help{margin-bottom:0}.search-block .input,.search-block select,.search-block .button{font-size:0.75rem;line-height:1.5;height:2.25em;border-radius:2px;border:1px solid transparent}.search-block .button{margin-left:0}.search-block .input{border-color:transparent;box-shadow:inset 0 1px 2px rgba(10,10,10,0.1);border-bottom-right-radius:0;border-top-right-radius:0;border:0;width:100%;max-width:100%}.search-block .control{position:relative}.search-block .select::after{position:absolute;display:block;z-index:4;top:50%;right:.65em;width:0.5em;height:0.5em;content:" ";border:3px solid #0068AC;border-radius:2px;border-right:0;border-top:0;transform:rotate(-45deg);transform-origin:center;pointer-events:none;margin-top:-1.125em}.search-block .select.is-small select{-webkit-appearance:none;-moz-appearance:none;appearance:none;display:block;width:100%;max-width:220px;height:27px;float:right;background-color:#ffffff;background-image:none;-ms-word-break:normal;word-break:normal;border-color:#ccc;box-shadow:inset 0 1px 2px rgba(10,10,10,0.1);border-radius:0}.search-block .select.is-small select::-ms-expand{display:none}.search-block .button{background-color:#711111;color:#FFF;border-color:transparent}.search-block .button:hover,.search-block .button:focus{background-color:#440A0A;color:#FFF}#header form select,#header form input{padding:0 0.5em}</style>
<link rel=canonical href=https://arxiv.org/abs/2408.08333>
<meta name=description content="Abstract page for arXiv paper 2408.08333: CodeMirage: Hallucinations in Code Generated by Large Language Models"><meta property=og:type content=website>
<meta property=og:site_name content=arXiv.org>
<meta property=og:title content="CodeMirage: Hallucinations in Code Generated by Large Language Models">
<meta property=og:url content=https://arxiv.org/abs/2408.08333v2>
<meta property=og:image content=/static/browse/0.3.4/images/arxiv-logo-fb.png>
<meta property=og:image:secure_url content=/static/browse/0.3.4/images/arxiv-logo-fb.png>
<meta property=og:image:width content=1200>
<meta property=og:image:height content=700>
<meta property=og:image:alt content="arXiv logo">
<meta property=og:description content="Large Language Models (LLMs) have shown promising potentials in program generation and no-code automation. However, LLMs are prone to generate hallucinations, i.e., they generate text which sounds plausible but is incorrect. Although there has been a recent surge in research on LLM hallucinations for text generation, similar hallucination phenomenon can happen in code generation. Sometimes the generated code can have syntactical or logical errors as well as more advanced issues like security vulnerabilities, memory leaks, etc. Given the wide adaptation of LLMs to enhance efficiency in code generation and development in general, it becomes imperative to investigate hallucinations in code generation. To the best of our knowledge, this is the first attempt at studying hallucinations in the code generated by LLMs. We start by introducing the code hallucination definition and a comprehensive taxonomy of code hallucination types. We propose the first benchmark CodeMirage dataset for code hallucinations. The benchmark contains 1,137 GPT-3.5 generated hallucinated code snippets for Python programming problems from two base datasets - HumanEval and MBPP. We then propose the methodology for code hallucination detection and experiment with open source LLMs such as CodeLLaMA as well as OpenAI's GPT-3.5 and GPT-4 models using one-shot prompt. We find that GPT-4 performs the best on HumanEval dataset and gives comparable results to the fine-tuned CodeBERT baseline on MBPP dataset. Towards the end, we discuss various mitigation strategies for code hallucinations and conclude our work.">
<meta name=twitter:site content=@arxiv>
<meta name=twitter:card content=summary>
<meta name=twitter:title content="CodeMirage: Hallucinations in Code Generated by Large Language Models">
<meta name=twitter:description content="Large Language Models (LLMs) have shown promising potentials in program generation and no-code automation. However, LLMs are prone to generate hallucinations, i.e., they generate text which sounds...">
<meta name=twitter:image content=https://static.arxiv.org/icons/twitter/arxiv-logo-twitter-square.png>
<meta name=twitter:image:alt content="arXiv logo">
<style media=screen>.visually-hidden{clip-path:inset(100%);clip:rect(1px,1px,1px,1px);height:1px;overflow:hidden;position:absolute;white-space:nowrap;width:1px}.button-and-tooltip{position:relative;display:inline}.button-and-tooltip button{background:transparent;border:none;box-shadow:none;position:relative;padding:0 0 0 6px}[role="tooltip"]{display:none;position:absolute;top:-6px;left:28px;width:250px;background-color:#f7fafc;border:1px solid black;padding:4px 10px;border-radius:5px}button:hover+[role="tooltip"],button:focus+[role="tooltip"]{display:block}td.tablecell.arxivdoi{display:inline-flex;align-items:center}@media screen and (max-width:480px){[role="tooltip"]{width:250px;left:-273px}}</style><style media=screen>.arrow{width:0;height:0;border-left:8px solid transparent;border-right:8px solid transparent}.bib-sort-arrow a:hover{text-decoration:none;cursor:pointer}.bib-authors a:after{content:","}.bib-authors a:last-of-type:after{content:""}h1.bib-header:before{border-top:2px solid #dfdfdf;content:"";margin:0 auto;position:absolute;top:50%;left:0;right:0;bottom:0;width:95%;z-index:-1}.bib-sidebar-msgs .msg:hover .hover{visibility:visible}@-webkit-keyframes pulsate{0%{-webkit-transform:scale(.1);transform:scale(.1);opacity:0}30%{opacity:1}60%{-webkit-transform:scale(.8);transform:scale(.8);opacity:0}}@keyframes pulsate{0%{-webkit-transform:scale(.1);transform:scale(.1);opacity:0}30%{opacity:1}60%{-webkit-transform:scale(.8);transform:scale(.8);opacity:0}}.tooltiptext:after{opacity:1;content:" ";position:absolute;top:100%;left:50%;margin-left:-5px;border-width:5px;border-style:solid;border-color:#000 transparent transparent}.modal-close:focus,.modal-close:hover{color:#000;text-decoration:none;cursor:pointer}.cite:hover{cursor:pointer}</style>
<meta name=citation_title content="CodeMirage: Hallucinations in Code Generated by Large Language Models"><meta name=citation_author content="Agarwal, Vibhor"><meta name=citation_author content="Pei, Yulong"><meta name=citation_author content="Alamir, Salwa"><meta name=citation_author content="Liu, Xiaomo"><meta name=citation_date content=2024/08/14><meta name=citation_online_date content=2025/07/08><meta name=citation_pdf_url content=https://arxiv.org/pdf/2408.08333><meta name=citation_arxiv_id content=2408.08333><meta name=citation_abstract content="Large Language Models (LLMs) have shown promising potentials in program generation and no-code automation. However, LLMs are prone to generate hallucinations, i.e., they generate text which sounds plausible but is incorrect. Although there has been a recent surge in research on LLM hallucinations for text generation, similar hallucination phenomenon can happen in code generation. Sometimes the generated code can have syntactical or logical errors as well as more advanced issues like security vulnerabilities, memory leaks, etc. Given the wide adaptation of LLMs to enhance efficiency in code generation and development in general, it becomes imperative to investigate hallucinations in code generation. To the best of our knowledge, this is the first attempt at studying hallucinations in the code generated by LLMs. We start by introducing the code hallucination definition and a comprehensive taxonomy of code hallucination types. We propose the first benchmark CodeMirage dataset for code hallucinations. The benchmark contains 1,137 GPT-3.5 generated hallucinated code snippets for Python programming problems from two base datasets - HumanEval and MBPP. We then propose the methodology for code hallucination detection and experiment with open source LLMs such as CodeLLaMA as well as OpenAI's GPT-3.5 and GPT-4 models using one-shot prompt. We find that GPT-4 performs the best on HumanEval dataset and gives comparable results to the fine-tuned CodeBERT baseline on MBPP dataset. Towards the end, we discuss various mitigation strategies for code hallucinations and conclude our work.">
<style>#MathJax_Message{position:fixed;left:1px;bottom:2px;background-color:#E6E6E6;border:1px solid #959595;margin:0px;padding:2px 8px;z-index:102;color:black;font-size:80%;width:auto;white-space:nowrap}</style><style>.MathJax_MenuDisabled:focus,.MathJax_MenuLabel:focus{background-color:#E8E8E8}.MathJax_ContextMenu:focus{outline:none}.MathJax_ContextMenu .MathJax_MenuItem:focus{outline:none}.MathJax_MenuClose:hover{color:white!important;border:2px solid #CCC!important}.MathJax_MenuClose:hover span{background-color:#CCC!important}.MathJax_MenuClose:hover:focus{outline:none}</style><meta name=referrer content=no-referrer><link rel=icon type=image/png sizes=32x32 href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAMAAABEpIrGAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAACr1BMVEUAAACzICWzICSyrKazrKWzqaKzraazR0mzTE6zaGizBQiz49uzZmSzEBazHySzsam0ICmzICWzICWzICWzICWzICWzICWzICWzICWzICWzICWzICWzICWzICWzICWzICWzrKWzrKWzICWzICWzICWzICWzICWzrKWzrKWzrKWzrKWzICWzICWzICWzICWzICWzrKWzrKWzrKWzrKWzICWzICWzICWzICWzrKWzrKWzrKWzrKWzICWzICWzICWzrKWzrKWzrKWzrKWzICWzICWzICWzICWzrKWzrKWzrKWzICWzICWzICWzHySz4dWzrKWzrKWzrKWzICWzICWzHySzgX2zrqezrKWzrKWzICWzPD+zrKWzrKWzrKWzICWzHySzjYmzr6izraazT1CzLjKzMDSzQUOzrKWzrqezgH2zKi6zHySzICWzrKWzrKWzcW+zICWzICWzrKWzrKWzYWGzICWzrKWzUVKzICWzrKWzrKWznZezQkSzICWzICWzrKWzraazkY2zHiOzICWzICWzjYmzm5WzhIGzODuzAACzHySzHSKzOz6zrqazs6yzICWzICWzODuzrKWzrKWzrKWzICWzICWzODuzo5yzrKWzrKWzICWzICWzICWzGyCzr6ezrKWzrKWzICWzICWzrKWzrKWzrKWzrKWzICWzICWzICWzrKWzrKWzrKWzrKWzICWzICWzICWzrKWzrKWzrKWzrKWzrKWzICWzICWzICWzrKWzrKWzrKWzrKWzICWzICWzICWzrKWzrKWzrKWzICWzICWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzrKWzICWzrKWzHySzkYyzrqezHiOzODuzJCmzqaOzISazraazpZ6zNTizkIuzi4ezraX///8A/VYjAAAA1HRSTlMAAAAAAAAAAAAAAAAAAAAAAAGA95kOB7X6kQ9W8fyaEQIBA3z9oRUllnUKDJz+qBgwxuwxGbquHD3SpQwt1CBL3cEeR+e7JFvm1mX0wSgEbO5HBYXFkfPyYaX++n0EHsH9lwtO8/NRJrr8/ttFGb7+3jpk/v6ecv6sK9j+/vVcQNf+8nUGSfD7dQMHivyVCmr2/vl3A0np28zvWS7V0joht+HJMR2vzCcLvygYp7IVc7QhFJ/9lQkSzhoRl/t2lu0UDY9XDSoLhvjgOwh89a4GB4L2i8A61lMAAAABYktHROQvYjspAAAAB3RJTUUH5wETDS455INCLAAAAgZJREFUOMt10/dfTXEYB/Dz6Ilst3lFueWmjDIysmdkF0Kyyd7Ze5WZcc2ErIhKISops6zs73Pda1zrH/EcoXPqe8+vn/fre57z+T5HURQwuHt4eoHi9AFvH2Fs6gu1nAJo1lz4+beAqkNc0BSAGhHY0ijMQa3+C8TgkNZttKJtOyFCw9r/FYgdOlKncC2Azl2EEF27/RGIEd2JevRE3Ri9erPw6cMCA/r2I+o/AFE/6MBBLCIHg6tpSBTR0HB9rophw1mMGDlqdDRRVEz1nMWYsX4sxsWOJ5owsWbOwjeOgXXSR1v85CmSnF8ydZpVCPun6TNm1pY2Wsdt1mwW1oQ5devJcsS58+YvUMXCRSC7Oly8hGjpZxZi2XKJQFyRSPRl5VcGYpVBBlYTkePbGvWItYGSd+C69arYsNEuNm2WDYG4ZauDyLbt+/YdO6X7Vb9BUjILx67dexpKe2i0d9/+HzYWKQcay3IwHDxkP/yT57AcOSq7Cjh2nIs+cfIXUeopyWVC2mn+vjNn08/xGecjaojKpbtwEVwvZbC4fKWagEx/zq9eA0DMyuZKc67rN65JrlGIvBtqP4g3b+VT4u07uqUtKBTmsKLK/hDvFt8jS4lJu/fepfc9H1T9OA8fxVPqY80YAE/KMjX9Y/nTZ/T8hWwz/4mXFa/o9RvnQMG3795/UKf4DUwCyzJ9eBcsAAAAJXRFWHRkYXRlOmNyZWF0ZQAyMDIzLTAxLTE5VDEzOjQ2OjU3KzAwOjAwqJ4w4AAAACV0RVh0ZGF0ZTptb2RpZnkAMjAyMy0wMS0xOVQxMzo0Njo1NyswMDowMNnDiFwAAABXelRYdFJhdyBwcm9maWxlIHR5cGUgaXB0YwAAeJzj8gwIcVYoKMpPy8xJ5VIAAyMLLmMLEyMTS5MUAxMgRIA0w2QDI7NUIMvY1MjEzMQcxAfLgEigSi4A6hcRdPJCNZUAAAAASUVORK5CYII="><style>.sf-hidden{display:none!important}</style><meta http-equiv=content-security-policy content="default-src 'none'; font-src 'self' data:; img-src 'self' data:; style-src 'unsafe-inline'; media-src 'self' data:; script-src 'unsafe-inline' data:; object-src 'self' data:; frame-src 'self' data:;"></head>
<body class=with-cu-identity><div id=MathJax_Message style=display:none></div>
<div class=flex-wrap-footer>
<header>
<a href=#content class=is-sr-only>Skip to main content</a>
<div class="columns is-vcentered is-hidden-mobile" id=cu-identity>
<div class=column id=cu-logo>
<a href=https://www.cornell.edu/><img src=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDIyLjEuMCwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxheWVyXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IgoJIHZpZXdCb3g9IjAgMCAyMDAuNyA0NSIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgMjAwLjcgNDU7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4KPHN0eWxlIHR5cGU9InRleHQvY3NzIj4KCS5zdDB7ZmlsbDojRkZGRkZGO30KCS5zdDF7ZmlsbDojRkZGRkZGO3N0cm9rZTojMDAwMDAwO3N0cm9rZS13aWR0aDowLjE1NjE7fQoJLnN0MntmaWxsOiNGRkZGRkY7c3Ryb2tlOiMwMDAwMDA7fQo8L3N0eWxlPgo8ZyBpZD0iTGF5ZXJfMl8xXyI+CjwvZz4KPGc+Cgk8ZyBpZD0iTGF5ZXJfMV8xXyI+CgkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTIyLjQsNDVDMTAsNDUsMCwzNC44LDAsMjIuNFMxMCwwLDIyLjQsMHMyMi40LDEwLDIyLjQsMjIuNEM0NC45LDM0LjgsMzQuOCw0NSwyMi40LDQ1eiBNMjIuNCwyLjUKCQkJYy0xMSwwLTIwLDktMjAsMjBzOSwyMCwyMCwyMHMyMC05LDIwLTIwQzQyLjQsMTEuNCwzMy41LDIuNSwyMi40LDIuNXoiLz4KCQk8cGF0aCBjbGFzcz0ic3QxIiBkPSJNMTcuMiwyNC45Ii8+CgkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTIyLjQsNDIuM2wtMC40LTAuMWMtMC41LTAuMi0xMy4yLTUuOC0xMy4yLTE1LjlWOC4xaDI3LjJ2MTguNGMwLDkuNy0xMi42LDE1LjMtMTMuMiwxNS42TDIyLjQsNDIuM3oKCQkJIE0xMC44LDkuOXYxNi4zYzAsOC4xLDkuNywxMy4xLDExLjgsMTQuMWMyLTEsMTEuOC02LjEsMTEuOC0xMy43VjEwSDEwLjhDMTAuOCwxMCwxMC44LDkuOSwxMC44LDkuOXoiLz4KCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNMTYuNywxOC44bC0wLjItMC4xYzAsMC0xLjUtMC45LTIuMy0xLjVjLTAuNy0wLjUtMS4xLTEuMS0xLjItMS42YzAtMC4xLDAtMS42LDAtNC41di0wLjZoNy43VjExCgkJCWMwLDIuOSwwLDQuNCwwLDQuNWMwLDAuNi0wLjQsMS4xLTEuMSwxLjZjLTEsMC42LTIuMywxLjUtMi40LDEuNUwxNi43LDE4Ljh6IE0xNCwxMS41YzAsMS42LDAsMy43LDAsMy45YzAsMC4xLDAuMiwwLjUsMC42LDAuNwoJCQljMC42LDAuNCwxLjUsMSwyLDEuMmMwLjUtMC4yLDEuNS0wLjksMi4xLTEuMmMwLjItMC4xLDAuNi0wLjQsMC42LTAuNmMwLTAuMiwwLTIuMiwwLTMuOUgxNEwxNCwxMS41TDE0LDExLjV6Ii8+CgkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTI4LjcsMTguOGwtMC4yLTAuMWMwLDAtMS41LTAuOS0yLjMtMS41Yy0wLjctMC41LTEuMS0xLjEtMS4yLTEuNmMwLTAuMSwwLTEuNiwwLTQuNXYtMC42aDcuN1YxMQoJCQljMCwyLjksMCw0LjQsMCw0LjVjMCwwLjYtMC40LDEuMS0xLjEsMS42Yy0xLDAuNi0yLjMsMS41LTIuNCwxLjVMMjguNywxOC44eiBNMjYsMTEuNWMwLDEuNiwwLDMuNywwLDMuOWMwLDAuMSwwLjIsMC41LDAuNiwwLjcKCQkJYzAuNiwwLjQsMS41LDEsMiwxLjJjMC41LTAuMiwxLjUtMC45LDIuMS0xLjJjMC4yLTAuMSwwLjYtMC40LDAuNi0wLjZjMC0wLjIsMC0yLjIsMC0zLjlIMjZMMjYsMTEuNUwyNiwxMS41eiIvPgoJCTxyZWN0IHg9IjkuMyIgeT0iMTkuMSIgY2xhc3M9InN0MCIgd2lkdGg9IjI2LjUiIGhlaWdodD0iMS42Ii8+CgkJPGc+CgkJCTxnPgoJCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTIyLjQsMzUuMmMtMC41LDAtMC43LTAuNC0wLjktMC41Yy0wLjEtMC4xLTAuMi0wLjItMC40LTAuMmMtMC43LDAtMS4yLDAtMS44LDAuMWMtMC42LDAtMS4yLDAuMS0yLjIsMC4xCgkJCQkJcy0xLjcsMC0xLjcsMGgtMC43VjIyLjNoMC43YzAuNSwwLDEuMSwwLDIuMSwwYzAuNSwwLDEtMC4xLDEuNi0wLjFjMC40LDAsMC43LTAuMSwxLjEtMC4xYzAuOS0wLjEsMS42LDAuMSwxLjcsMC4xCgkJCQkJYzAuMiwwLDAuNCwwLjEsMC42LDAuMmMwLjEtMC4xLDAuNC0wLjEsMC42LTAuMmMwLDAsMC45LTAuMSwxLjctMC4xYzAuNCwwLDAuNywwLjEsMS4xLDAuMWMwLjYsMC4xLDEuMSwwLjEsMS42LDAuMQoJCQkJCWMxLDAsMS42LDAsMi4xLDBoMC43djEyLjRoLTAuN2MwLDAtMC43LDAtMS43LDBjLTEsMC0xLjYtMC4xLTIuMi0wLjFjLTAuNiwwLTEuMS0wLjEtMS44LTAuMWMtMC4yLDAtMC4yLDAtMC40LDAuMgoJCQkJCUMyMy4yLDM1LDIyLjksMzUuMiwyMi40LDM1LjJ6IE0yMS4yLDMzLjFjMC42LDAsMS4xLDAuMiwxLjQsMC41YzAuMi0wLjIsMC43LTAuNSwxLjQtMC41YzAuNywwLDEuNCwwLDIsMC4xCgkJCQkJYzAuNiwwLDEuMiwwLjEsMi4xLDAuMWMwLjQsMCwwLjYsMCwwLjksMHYtOS41Yy0wLjQsMC0wLjksMC0xLjQsMGMtMC41LDAtMS4xLTAuMS0xLjctMC4xYy0wLjQsMC0wLjctMC4xLTEuMS0wLjEKCQkJCQljLTAuNi0wLjEtMS4yLDAtMS4yLDBjLTAuMSwwLTAuMiwwLjEtMC4yLDAuMXMwLDAsMC4xLTAuMWwtMC43LTAuMWwtMC43LDAuMWMwLDAuMSwwLDAuMSwwLjEsMC4xYzAsMCwwLDAtMC4yLTAuMWwwLDAKCQkJCQljMCwwLTAuNi0wLjEtMS4yLDBjLTAuNCwwLTAuNywwLjEtMS4xLDAuMWMtMC42LDAuMS0xLjIsMC4xLTEuNywwLjFjLTAuNiwwLTEsMC0xLjQsMHY5LjVjMC4yLDAsMC42LDAsMC45LDAKCQkJCQljMC45LDAsMS41LTAuMSwyLjEtMC4xQzE5LjksMzMuMSwyMC40LDMzLjEsMjEuMiwzMy4xeiIvPgoJCQk8L2c+CgkJPC9nPgoJCTxyZWN0IHg9IjEzLjQiIHk9IjEyLjgiIGNsYXNzPSJzdDAiIHdpZHRoPSI2LjQiIGhlaWdodD0iMS4xIi8+CgkJPHJlY3QgeD0iMjEuOCIgeT0iMTkuNSIgY2xhc3M9InN0MCIgd2lkdGg9IjEuNSIgaGVpZ2h0PSIyMS44Ii8+CgkJPHBvbHlnb24gY2xhc3M9InN0MCIgcG9pbnRzPSIzMS40LDE1LjIgMjguNiwxMy40IDI2LDE1LjIgMjUuMywxNC4zIDI4LjYsMTIgMzIsMTQuMyAJCSIvPgoJCTxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik0yOC41LDE1LjMiLz4KCQk8cmVjdCB4PSIxNy4yIiB5PSIyNS4xIiBjbGFzcz0ic3QwIiB3aWR0aD0iMy4yIiBoZWlnaHQ9IjEuMSIvPgoJCTxyZWN0IHg9IjE3LjIiIHk9IjI3LjciIGNsYXNzPSJzdDAiIHdpZHRoPSIzLjIiIGhlaWdodD0iMS4xIi8+CgkJPHJlY3QgeD0iMTcuMiIgeT0iMzAuMyIgY2xhc3M9InN0MCIgd2lkdGg9IjMuMiIgaGVpZ2h0PSIxLjEiLz4KCQk8cmVjdCB4PSIyNC4zIiB5PSIyNS4xIiBjbGFzcz0ic3QwIiB3aWR0aD0iMy4yIiBoZWlnaHQ9IjEuMSIvPgoJCTxyZWN0IHg9IjI0LjMiIHk9IjI3LjciIGNsYXNzPSJzdDAiIHdpZHRoPSIzLjIiIGhlaWdodD0iMS4xIi8+CgkJPHJlY3QgeD0iMjQuMyIgeT0iMzAuMyIgY2xhc3M9InN0MCIgd2lkdGg9IjMuMiIgaGVpZ2h0PSIxLjEiLz4KCTwvZz4KCTxnIGlkPSJMYXllcl8zIj4KCQk8Zz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTY1LjEsMjguN2MtMS4xLDAuNy0zLjEsMS4xLTQuMywxLjFjLTQuNywwLTcuOC0yLjctNy44LTcuMWMwLTIuMiwwLjktNCwyLjQtNS4zYzEuNS0xLjIsMy40LTEuOCw1LjYtMS44CgkJCQljMS44LDAsMy42LDAuNSw0LjUsMC45Yy0wLjIsMS0wLjQsMi0wLjQsMi45aC0wLjZ2LTEuNWMwLTAuNS0wLjctMC45LTEuNy0xLjJjLTAuNi0wLjItMS41LTAuNC0yLjItMC40Yy0zLjcsMC01LjYsMi43LTUuNiw2CgkJCQljMCwzLjksMi42LDYuNCw2LjUsNi40YzEuNSwwLDMuMS0wLjUsMy45LTEuMmwwLjEsMC4yTDY1LjEsMjguN3oiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTcwLDI5LjdjLTIuNCwwLTQuMi0yLTQuMi00LjVjMC0yLjksMS44LTUsNS01YzIuNCwwLDQuNCwyLDQuNCw0LjRjMCwyLjktMi4xLDUuMi01LjIsNS4yTDcwLDI5LjdMNzAsMjkuNwoJCQkJTDcwLDI5LjdMNzAsMjkuN3ogTTY3LjcsMjQuM2MwLDIuMSwwLjcsNC44LDMuMyw0LjhjMS44LDAsMi42LTEuOCwyLjYtMy42YzAtMi42LTEuMi00LjctMy4xLTQuN0M2OC4zLDIwLjcsNjcuNywyMi40LDY3LjcsMjQuMwoJCQkJeiIvPgoJCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNNzYuOCwyMi45YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNWMxLjEtMC4xLDIuMy0wLjUsMy4xLTAuOWwwLjEsMC4xdjJsMCwwCgkJCQlsMS4xLTEuMmMwLjUtMC42LDEuMS0wLjksMS42LTAuOWMwLjQsMCwwLjcsMC4xLDAuOSwwLjJsLTAuMSwyLjNoLTAuNGMtMC4yLTAuNS0wLjYtMC45LTEuNC0wLjljLTAuOSwwLTEuNywwLjktMS43LDIuMnYzLjQKCQkJCWMwLDAuNSwwLDEsMC4xLDEuMWMwLjEsMC4xLDAuNCwwLjIsMC45LDAuMmgxdjAuNmMtMSwwLTEuOCwwLTIuOCwwYy0xLDAtMS43LDAtMi40LDB2LTAuNmgwLjljMC40LDAsMC41LTAuMSwwLjYtMC4yCgkJCQljMC4xLTAuMiwwLjEtMC42LDAuMS0xLjFMNzYuOCwyMi45TDc2LjgsMjIuOUw3Ni44LDIyLjlMNzYuOCwyMi45TDc2LjgsMjIuOXoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTg1LjgsMjcuM2MwLDAuNSwwLDEsMC4xLDEuMWMwLjEsMC4xLDAuMiwwLjIsMC42LDAuMmgwLjl2MC42Yy0wLjcsMC0xLjUsMC0yLjQsMHMtMS43LDAtMi40LDB2LTAuNmgwLjkKCQkJCWMwLjQsMCwwLjUtMC4xLDAuNi0wLjJjMC4xLTAuMiwwLjEtMC42LDAuMS0xLjF2LTQuNGMwLTAuNiwwLTEuMS0wLjEtMS4yYy0wLjEtMC4xLTAuNC0wLjItMC43LTAuMmgtMC42di0wLjUKCQkJCWMxLjEtMC4xLDIuMy0wLjUsMy4xLTAuOWwwLjEsMC4xdjEuN2wwLDBsMS41LTEuMmMwLjUtMC40LDAuOS0wLjUsMS42LTAuNWMyLjMsMCwzLjIsMS4yLDMuMiwzLjR2My44YzAsMC41LDAsMSwwLjEsMS4xCgkJCQljMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC41LDAtMS4xLDAtMS42LDBjLTAuNSwwLTEuMSwwLTEuNiwwdi01LjVjMC0xLjYtMS0yLjQtMi4yLTIuNGMtMS4yLDAtMi40LDEtMi40LDEuOAoJCQkJTDg1LjgsMjcuM0w4NS44LDI3LjN6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xMDEuOSwyMy43YzAsMC4yLDAsMC41LDAsMC45Yy0xLjIsMC4yLTIuMywwLjItMy42LDAuMmgtMi44YzAsMi4zLDEuMSwzLjksMy42LDMuOWMxLDAsMS43LTAuNSwyLjQtMQoJCQkJbDAuMiwwLjJsLTAuNCwwLjZjLTAuMSwwLjItMS44LDEuMS0zLjIsMS4xYy0yLjksMC00LjUtMS44LTQuNS00LjVjMC0yLDAuNS0zLjEsMS41LTMuOGMwLjQtMC4yLDAuNy0wLjYsMS4yLTAuNwoJCQkJYzAuNy0wLjQsMS4yLTAuNSwyLTAuNUMxMDAuMywyMC4xLDEwMS45LDIxLjUsMTAxLjksMjMuN3ogTTk1LjUsMjMuOFYyNGMwLjksMC4xLDEuNywwLjEsMi40LDAuMWMwLjcsMCwxLjUsMCwyLjIsMAoJCQkJYzAtMS44LTAuNi0zLjMtMi4xLTMuM0M5Ni42LDIwLjcsOTUuNSwyMS44LDk1LjUsMjMuOHoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTEwMy43LDE3LjJjMC0wLjUsMC0wLjktMC41LTAuOWgtMS4xdi0wLjVjMS0wLjEsMi4xLTAuNCwzLjEtMC43bDAuMSwwLjF2MTIuMWMwLDAuNSwwLDEsMC4xLDEuMQoJCQkJYzAuMSwwLjEsMC4yLDAuMiwwLjYsMC4yaDAuOXYwLjZjLTAuNywwLTEuNSwwLTIuNCwwcy0xLjcsMC0yLjQsMHYtMC42aDAuOWMwLjQsMCwwLjUtMC4xLDAuNi0wLjJjMC4xLTAuMiwwLjEtMC42LDAuMS0xLjEKCQkJCUwxMDMuNywxNy4yTDEwMy43LDE3LjJ6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xMDguNywxNy4yYzAtMC41LDAtMC45LTAuNS0wLjlIMTA3di0wLjVjMS0wLjEsMi4xLTAuNCwzLjEtMC43bDAuMSwwLjF2MTIuMWMwLDAuNSwwLDEsMC4xLDEuMQoJCQkJYzAuMSwwLjEsMC4yLDAuMiwwLjYsMC4yaDAuOXYwLjZjLTAuNywwLTEuNSwwLTIuNCwwYy0xLDAtMS43LDAtMi40LDB2LTAuNmgwLjljMC40LDAsMC41LTAuMSwwLjYtMC4yYzAuMS0wLjIsMC4xLTAuNiwwLjEtMS4xCgkJCQlMMTA4LjcsMTcuMkwxMDguNywxNy4yeiIvPgoJCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNMTE3LjgsMTguMmMwLTAuNywwLTEuMi0wLjEtMS41cy0wLjQtMC4yLTAuOS0wLjJoLTF2LTAuNmMxLDAsMiwwLDIuOSwwYzAuOSwwLDEuOCwwLDIuOCwwdjAuNmgtMQoJCQkJYy0wLjUsMC0wLjcsMC4xLTAuOSwwLjJjLTAuMSwwLjItMC4xLDAuNy0wLjEsMS41djYuN2MwLDIuOCwxLjUsMy42LDQsMy42YzIuMSwwLDQtMC45LDQtNHYtNi4zYzAtMC43LDAtMS4yLTAuMS0xLjUKCQkJCWMtMC4xLTAuMi0wLjQtMC4yLTAuOS0wLjJoLTAuOXYtMC42YzAuNywwLDEuNiwwLDIuMywwYzAuNywwLDEuNSwwLDIuMywwdjAuNmgtMC45Yy0wLjUsMC0wLjcsMC4xLTAuOSwwLjIKCQkJCWMtMC4xLDAuMi0wLjEsMC43LTAuMSwxLjV2NS42YzAsNC4yLTEuNiw1LjktNS41LDUuOWMtMy4zLDAtNS4zLTEtNS4zLTQuNUwxMTcuOCwxOC4yTDExNy44LDE4LjJMMTE3LjgsMTguMnoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTEzMy4yLDI3LjNjMCwwLjUsMCwxLDAuMSwxLjFjMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC43LDAtMS41LDAtMi40LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42CgkJCQloMC45YzAuNCwwLDAuNS0wLjEsMC42LTAuMmMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMXYtNC40YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNQoJCQkJYzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjF2MS43bDAsMGwxLjUtMS4yYzAuNS0wLjQsMC45LTAuNSwxLjYtMC41YzIuMywwLDMuMiwxLjIsMy4yLDMuNHYzLjhjMCwwLjUsMCwxLDAuMSwxLjEKCQkJCWMwLjEsMC4xLDAuMiwwLjIsMC42LDAuMmgwLjl2MC42Yy0wLjUsMC0xLjEsMC0xLjYsMGMtMC41LDAtMS4xLDAtMS42LDB2LTUuNWMwLTEuNi0xLTIuNC0yLjItMi40Yy0xLjIsMC0yLjQsMS0yLjQsMS44CgkJCQlMMTMzLjIsMjcuM0wxMzMuMiwyNy4zTDEzMy4yLDI3LjNMMTMzLjIsMjcuM3oiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE0NC45LDI3LjNjMCwwLjUsMCwxLDAuMSwxLjFjMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC43LDAtMS41LDAtMi40LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42CgkJCQloMC45YzAuNCwwLDAuNS0wLjEsMC42LTAuMmMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMXYtNC40YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNQoJCQkJYzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjFMMTQ0LjksMjcuM0wxNDQuOSwyNy4zTDE0NC45LDI3LjNMMTQ0LjksMjcuM3ogTTE0NS4xLDE2LjljMCwwLjYtMC41LDEuMS0xLjEsMS4xCgkJCQljLTAuNiwwLTEuMS0wLjUtMS4xLTEuMXMwLjUtMS4xLDEuMS0xLjFDMTQ0LjYsMTUuOCwxNDUuMSwxNi4zLDE0NS4xLDE2Ljl6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xNTIuMywyNy4zYy0wLjQsMC43LTAuNSwxLjUtMC45LDIuMWgtMWwtMy40LThjLTAuMS0wLjItMC4yLTAuNi0wLjYtMC42aC0wLjZ2LTAuNWMwLjcsMCwxLjUsMCwyLjMsMAoJCQkJYzAuNywwLDEuNSwwLDIuMywwdjAuNWgtMWMtMC40LDAtMC41LDAuMS0wLjUsMC40YzAsMC4xLDAsMC40LDAuMSwwLjdsMi4zLDUuNmMwLjQtMC45LDAuOS0xLjgsMS4yLTIuN2wwLjktMi4xCgkJCQljMC4yLTAuNiwwLjQtMS4xLDAuNC0xLjVjMC0wLjQtMC4xLTAuNS0wLjUtMC41aC0wLjl2LTAuNWMwLjYsMCwxLjIsMCwxLjgsMGMwLjYsMCwxLjIsMCwxLjgsMHYwLjVoLTAuNmMtMC41LDAtMC45LDAuNy0xLjEsMS40CgkJCQlMMTUyLjMsMjcuM3oiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE2NC4xLDIzLjdjMCwwLjIsMCwwLjUsMCwwLjljLTEuMiwwLjItMi4zLDAuMi0zLjYsMC4yaC0yLjhjMCwyLjMsMS4xLDMuOSwzLjYsMy45YzEsMCwxLjctMC41LDIuNC0xCgkJCQlsMC4yLDAuMmwtMC40LDAuNmMtMC4xLDAuMi0xLjgsMS4xLTMuMiwxLjFjLTIuOSwwLTQuNS0xLjgtNC41LTQuNWMwLTIsMC41LTMuMSwxLjUtMy44YzAuNC0wLjIsMC43LTAuNiwxLjItMC43CgkJCQljMC43LTAuNCwxLjItMC41LDItMC41QzE2Mi41LDIwLjEsMTY0LjEsMjEuNSwxNjQuMSwyMy43eiBNMTU3LjYsMjMuOFYyNGMwLjksMC4xLDEuNywwLjEsMi40LDAuMXMxLjUsMCwyLjIsMAoJCQkJYzAtMS44LTAuNi0zLjMtMi4xLTMuM0MxNTguOCwyMC43LDE1Ny42LDIxLjgsMTU3LjYsMjMuOHoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE2Ni4zLDIyLjljMC0wLjYsMC0xLjEtMC4xLTEuMmMtMC4xLTAuMS0wLjQtMC4yLTAuNy0wLjJoLTAuN3YtMC41YzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjF2MmwwLDAKCQkJCWwxLjEtMS4yYzAuNS0wLjYsMS4xLTAuOSwxLjYtMC45YzAuNCwwLDAuNywwLjEsMC45LDAuMmwtMC4xLDIuM2gtMC40Yy0wLjItMC41LTAuNi0wLjktMS40LTAuOWMtMC45LDAtMS43LDAuOS0xLjcsMi4ydjMuNAoJCQkJYzAsMC41LDAsMSwwLjEsMS4xYzAuMSwwLjEsMC40LDAuMiwwLjksMC4yaDF2MC42Yy0xLDAtMS44LDAtMi44LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42aDAuOWMwLjQsMCwwLjUtMC4xLDAuNi0wLjIKCQkJCWMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMUwxNjYuMywyMi45TDE2Ni4zLDIyLjlMMTY2LjMsMjIuOUwxNjYuMywyMi45eiIvPgoJCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNMTczLDI2LjV2MC45YzAsMS4yLDEuNCwxLjcsMi42LDEuN2MxLjIsMCwyLjMtMC43LDIuMy0xLjhjMC0wLjYtMC40LTEuMS0xLTEuM2MtMC45LTAuMi0yLTAuNS0yLjktMC43CgkJCQljLTEtMC40LTEuNy0xLTEuNy0yLjFjMC0yLjEsMS44LTIuOCwzLjctMi44YzEsMCwxLjcsMC4yLDIuNiwwLjVjMCwwLjctMC4xLDEuNS0wLjEsMi4yaC0wLjV2LTAuNWMwLTEtMS4xLTEuNi0yLjMtMS42CgkJCQljLTEuNywwLTIsMS0yLDEuNmMwLDAuOSwwLjYsMS40LDIuMSwxLjZjMi4zLDAuNCwzLjQsMSwzLjQsMi40YzAsMi4yLTIuMiwzLjMtNC4zLDMuM2MtMSwwLTEuOC0wLjEtMi43LTAuNQoJCQkJYzAuMi0wLjksMC4yLTEuOCwwLjItMi43aDAuNkwxNzMsMjYuNUwxNzMsMjYuNUwxNzMsMjYuNUwxNzMsMjYuNXoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE4My4yLDI3LjNjMCwwLjUsMCwxLDAuMSwxLjFjMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC43LDAtMS41LDAtMi40LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42CgkJCQloMC45YzAuNCwwLDAuNS0wLjEsMC42LTAuMmMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMXYtNC40YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNQoJCQkJYzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjFMMTgzLjIsMjcuM0wxODMuMiwyNy4zTDE4My4yLDI3LjNMMTgzLjIsMjcuM3ogTTE4My40LDE2LjljMCwwLjYtMC41LDEuMS0xLjEsMS4xCgkJCQljLTAuNiwwLTEuMS0wLjUtMS4xLTEuMXMwLjUtMS4xLDEuMS0xLjFDMTgyLjgsMTUuOCwxODMuNCwxNi4zLDE4My40LDE2Ljl6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xODQuNSwyMnYtMC40bDEuNS0wLjd2LTEuM2MwLTAuNSwwLTEtMC4xLTEuNmMwLjctMC4yLDEuNC0wLjUsMS43LTAuN2wwLjIsMC4yYy0wLjEsMC45LTAuMiwyLTAuMiwyLjhWMjEKCQkJCWwyLjctMC4xbC0wLjEsMS4xaC0yLjR2NS4yYzAsMC45LDAuMiwxLjQsMS4xLDEuNGMwLjUsMCwwLjktMC4yLDEuMS0wLjRsMC4yLDAuNGwtMSwxYy0wLjEsMC4yLTAuOSwwLjItMS4yLDAuMgoJCQkJYy0xLDAtMi0wLjUtMi0yLjF2LTUuN0wxODQuNSwyMnoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE5OC4zLDIyYzAuMS0wLjIsMC4xLTAuNCwwLjEtMC41YzAtMC40LTAuMi0wLjUtMC45LTAuNUgxOTd2LTAuNWMwLjYsMCwxLjIsMCwxLjgsMGMwLjYsMCwxLjIsMCwxLjgsMFYyMQoJCQkJaC0wLjVjLTAuNSwwLTAuOSwwLjYtMS42LDIuM2wtMy45LDkuMmMtMC42LDEuNS0xLjMsMi40LTIuOSwyLjRjLTAuNCwwLTAuNy0wLjEtMS0wLjJsMC41LTEuNWgwLjJjMC4yLDAuMiwwLjcsMC41LDEsMC41bDAsMAoJCQkJYzEuMS0wLjEsMS43LTEuNywyLjEtMi42bDAuNS0xLjJsLTMuMi04Yy0wLjQtMC43LTAuNi0xLTEtMWgtMC40di0wLjVjMC43LDAsMS41LDAsMi4zLDBjMC43LDAsMS41LDAsMi4zLDBWMjFoLTAuNwoJCQkJYy0wLjQsMC0wLjYsMC4xLTAuNiwwLjVjMCwwLjIsMCwwLjUsMC4xLDAuN2wyLjIsNS40TDE5OC4zLDIyeiIvPgoJCTwvZz4KCTwvZz4KPC9nPgo8L3N2Zz4K alt="Cornell University"></a>
</div><div class=column id=support-ack>
<span id=support-ack-url>We gratefully acknowledge support from<br>the Simons Foundation, <a href=https://info.arxiv.org/about/ourmembers.html>Stockholm University</a>, and all contributors.</span>
<a href=https://info.arxiv.org/about/donate.html class=btn-header-donate>Donate</a>
</div>
</div>
<div id=header class=is-hidden-mobile>
<a aria-hidden=true tabindex=-1 href=https://arxiv.org/IgnoreMe></a>
<div class="header-breadcrumbs is-hidden-mobile">
<a href=https://arxiv.org/><img src=data:image/svg+xml;base64,PHN2ZyBpZD0icHJpbWFyeV9sb2dvXy1fc2luZ2xlX2NvbG9yXy1fd2hpdGUiIGRhdGEtbmFtZT0icHJpbWFyeSBsb2dvIC0gc2luZ2xlIGNvbG9yIC0gd2hpdGUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgdmlld0JveD0iMCAwIDI0Ni45NzggMTEwLjExOSI+PHBhdGggZD0iTTQ5Mi45NzYsMjY5LjVsMjQuMzYtMjkuODljMS40OTItMS45ODksMi4yLTMuMDMsMS40OTItNC43MjNhNS4xNDIsNS4xNDIsMCwwLDAtNC40ODEtMy4xNjFoMGE0LjAyNCw0LjAyNCwwLDAsMC0zLjAwOCwxLjEwOEw0ODUuMiwyNjEuMDk0WiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTM1OC4xNjUgLTIyMy4yNykiIGZpbGw9IiNmZmYiLz48cGF0aCBkPSJNNTI2LjI3MywzMjUuMzQxLDQ5My45MSwyODcuMDU4bC0uOTcyLDEuMDMzLTcuNzg5LTkuMjE0LTcuNzQzLTkuMzU3LTQuNjk1LDUuMDc2YTQuNzY5LDQuNzY5LDAsMCwwLC4wMTUsNi41M0w1MjAuNTEyLDMzMi4yYTMuOTEzLDMuOTEzLDAsMCwwLDMuMTM3LDEuMTkyLDQuMzk0LDQuMzk0LDAsMCwwLDQuMDI3LTIuODE4QzUyOC40LDMyOC44NDQsNTI3LjYsMzI3LjEzMyw1MjYuMjczLDMyNS4zNDFaIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMzU4LjE2NSAtMjIzLjI3KSIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Ik00NzkuMjE1LDI4OC4wODdsNi4wNTIsNi40ODVMNDU4LjcxNCwzMjIuN2EyLjk4LDIuOTgsMCwwLDEtMi4yNzUsMS4xOTQsMy40NDksMy40NDksMCwwLDEtMy4yNDEtMi4xNDRjLS41MTMtMS4yMzEuMTY2LTMuMTUsMS4xMjItNC4xNjhsLjAyMy0uMDI0LjAyMS0uMDI2LDI0Ljg1MS0yOS40NDhtLS4wNDctMS44ODItMjUuNzYsMzAuNTI0Yy0xLjI4NiwxLjM3Mi0yLjA4NCwzLjc3Ny0xLjM2NSw1LjVhNC43MDUsNC43MDUsMCwwLDAsNC40LDIuOTE0LDQuMTkxLDQuMTkxLDAsMCwwLDMuMTYxLTEuNTYzbDI3LjM4Mi0yOS4wMDctNy44MTQtOC4zNzJaIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMzU4LjE2NSAtMjIzLjI3KSIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Ik00MjcuNTcxLDI1NS4xNTRjMS44NTksMCwzLjEsMS4yNCwzLjk4NSwzLjQ1MywxLjA2Mi0yLjIxMywyLjU2OC0zLjQ1Myw0LjY5NC0zLjQ1M2gxNC44NzhhNC4wNjIsNC4wNjIsMCwwLDEsNC4wNzQsNC4wNzR2Ny44MjhjMCwyLjY1Ni0xLjMyNyw0LjA3NC00LjA3NCw0LjA3NC0yLjY1NiwwLTQuMDc0LTEuNDE4LTQuMDc0LTQuMDc0VjI2My4zSDQzNi41MTVhMi40MTEsMi40MTEsMCwwLDAtMi42NTYsMi43NDV2MjcuMTg4aDEwLjAwN2MyLjY1OCwwLDQuMDc0LDEuMzI5LDQuMDc0LDQuMDc0cy0xLjQxNiw0LjA3NC00LjA3NCw0LjA3NGgtMjYuMzljLTIuNjU5LDAtMy45ODYtMS4zMjgtMy45ODYtNC4wNzRzMS4zMjctNC4wNzQsMy45ODYtNC4wNzRoOC4yMzZWMjYzLjNoLTcuMjYzYy0yLjY1NiwwLTMuOTg1LTEuMzI5LTMuOTg1LTQuMDc0LDAtMi42NTgsMS4zMjktNC4wNzQsMy45ODUtNC4wNzRaIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMzU4LjE2NSAtMjIzLjI3KSIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Ik01MzkuMjMzLDI1NS4xNTRjMi42NTYsMCw0LjA3NCwxLjQxNiw0LjA3NCw0LjA3NHYzNC4wMDdoMTAuMWMyLjc0NiwwLDQuMDc0LDEuMzI5LDQuMDc0LDQuMDc0cy0xLjMyOCw0LjA3NC00LjA3NCw0LjA3NEg1MjQuOGMtMi42NTYsMC00LjA3NC0xLjMyOC00LjA3NC00LjA3NHMxLjQxOC00LjA3NCw0LjA3NC00LjA3NGgxMC4zNjJWMjYzLjNoLTguNTMzYy0yLjc0NCwwLTQuMDczLTEuMzI5LTQuMDczLTQuMDc0LDAtMi42NTgsMS4zMjktNC4wNzQsNC4wNzMtNC4wNzRabTQuMjItMTcuNjE1YTUuODU5LDUuODU5LDAsMSwxLTUuODE5LTUuODE5QTUuOSw1LjksMCwwLDEsNTQzLjQ1MywyMzcuNTM5WiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTM1OC4xNjUgLTIyMy4yNykiIGZpbGw9IiNmZmYiLz48cGF0aCBkPSJNNjA1LjE0MywyNTkuMjI4YTQuNTg5LDQuNTg5LDAsMCwxLS4yNjcsMS41OTRMNTkwLDI5OC45YTMuNzIyLDMuNzIyLDAsMCwxLTMuNzIxLDIuNDhoLTUuOTMzYTMuNjg5LDMuNjg5LDAsMCwxLTMuODA4LTIuNDhsLTE1LjA1NS0zOC4wODFhMy4yMywzLjIzLDAsMCwxLS4zNTUtMS41OTQsNC4wODQsNC4wODQsMCwwLDEsNC4xNjQtNC4wNzQsMy44LDMuOCwwLDAsMSwzLjcxOCwyLjY1NmwxNC4zNDgsMzYuMTM0LDEzLjktMzYuMTM0YTMuOCwzLjgsMCwwLDEsMy43Mi0yLjY1NkE0LjA4NCw0LjA4NCwwLDAsMSw2MDUuMTQzLDI1OS4yMjhaIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMzU4LjE2NSAtMjIzLjI3KSIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Ik0zOTAuNjEsMjU1LjE1NGM1LjAxOCwwLDguMjA2LDMuMzEyLDguMjA2LDguNHYzNy44MzFIMzYzLjMwOGE0LjgxMyw0LjgxMywwLDAsMS01LjE0My00LjkyOVYyODMuNDI3YTguMjU2LDguMjU2LDAsMCwxLDctOC4xNDhsMjUuNTA3LTMuNTcydi04LjRIMzYyLjMwNmE0LjAxNCw0LjAxNCwwLDAsMS00LjE0MS00LjA3NGMwLTIuODcsMi4xNDMtNC4wNzQsNC4zNTUtNC4wNzRabS4wNTksMzguMDgxVjI3OS45NDJsLTI0LjM1NCwzLjR2OS45WiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTM1OC4xNjUgLTIyMy4yNykiIGZpbGw9IiNmZmYiLz48cGF0aCBkPSJNNDQ4LjUzOCwyMjQuNTJoLjA3N2MxLC4wMjQsMi4yMzYsMS4yNDUsMi41ODksMS42NjlsLjAyMy4wMjguMDI0LjAyNiw0Ni42NjQsNTAuNDMzYTMuMTczLDMuMTczLDAsMCwxLS4wMzQsNC4zMzZsLTQuODkzLDUuMi02Ljg3Ni04LjEzNEw0NDYuNjUyLDIzMC40Yy0xLjUwOC0yLjE2Ni0xLjYxNy0yLjgzNi0xLjE5MS0zLjg1OGEzLjM1MywzLjM1MywwLDAsMSwzLjA3Ny0yLjAybTAtMS4yNWE0LjYwNiw0LjYwNiwwLDAsMC00LjIzMSwyLjc4OWMtLjcwNSwxLjY5Mi0uMiwyLjg4LDEuMzQ5LDUuMWwzOS40OTMsNDcuNzIyLDcuNzg5LDkuMjE0LDUuODUzLTYuMjIxYTQuNDE3LDQuNDE3LDAsMCwwLC4wNDItNi4wNDJMNDUyLjE2OSwyMjUuNHMtMS43MTMtMi4wOC0zLjUyNC0yLjEyNFoiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0zNTguMTY1IC0yMjMuMjcpIiBmaWxsPSIjZmZmIi8+PC9zdmc+ alt="arxiv logo" style=height:40px></a> <span>&gt;</span> <a href=https://arxiv.org/list/cs/recent>cs</a> <span>&gt;</span> arXiv:2408.08333
</div>
<div class="columns is-vcentered is-mobile" style=justify-content:flex-end>
</div>
<div class="search-block level-right">
<form class="level-item mini-search" method=GET action=https://arxiv.org/search>
<div class="field has-addons">
<div class=control>
<input class="input is-small" type=text name=query placeholder=Search... aria-label="Search term or terms" value>
<p class=help><a href=https://info.arxiv.org/help>Help</a> | <a href=https://arxiv.org/search/advanced>Advanced Search</a></p>
</div>
<div class=control>
<div class="select is-small">
<select name=searchtype aria-label="Field to search">
<option value=all selected>All fields</option>
<option value=title>Title</option>
<option value=author>Author</option>
<option value=abstract>Abstract</option>
<option value=comments>Comments</option>
<option value=journal_ref>Journal reference</option>
<option value=acm_class>ACM classification</option>
<option value=msc_class>MSC classification</option>
<option value=report_num>Report number</option>
<option value=paper_id>arXiv identifier</option>
<option value=doi>DOI</option>
<option value=orcid>ORCID</option>
<option value=author_id>arXiv author ID</option>
<option value=help>Help pages</option>
<option value=full_text>Full text</option>
</select>
</div>
</div>
<button class="button is-small is-cul-darker">Search</button>
</div>
</form>
</div>
</div>
<div class="mobile-header sf-hidden">
<div class="columns is-mobile sf-hidden">
<div class="column logo-cornell sf-hidden"><a href=https://www.cornell.edu/ class=sf-hidden>
<picture class=sf-hidden>
<source media="(min-width: 501px)" srcset="data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDIyLjEuMCwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxheWVyXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IgoJIHZpZXdCb3g9IjAgMCAyMDAuNyA0NSIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgMjAwLjcgNDU7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4KPHN0eWxlIHR5cGU9InRleHQvY3NzIj4KCS5zdDB7ZmlsbDojRkZGRkZGO30KCS5zdDF7ZmlsbDojRkZGRkZGO3N0cm9rZTojMDAwMDAwO3N0cm9rZS13aWR0aDowLjE1NjE7fQoJLnN0MntmaWxsOiNGRkZGRkY7c3Ryb2tlOiMwMDAwMDA7fQo8L3N0eWxlPgo8ZyBpZD0iTGF5ZXJfMl8xXyI+CjwvZz4KPGc+Cgk8ZyBpZD0iTGF5ZXJfMV8xXyI+CgkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTIyLjQsNDVDMTAsNDUsMCwzNC44LDAsMjIuNFMxMCwwLDIyLjQsMHMyMi40LDEwLDIyLjQsMjIuNEM0NC45LDM0LjgsMzQuOCw0NSwyMi40LDQ1eiBNMjIuNCwyLjUKCQkJYy0xMSwwLTIwLDktMjAsMjBzOSwyMCwyMCwyMHMyMC05LDIwLTIwQzQyLjQsMTEuNCwzMy41LDIuNSwyMi40LDIuNXoiLz4KCQk8cGF0aCBjbGFzcz0ic3QxIiBkPSJNMTcuMiwyNC45Ii8+CgkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTIyLjQsNDIuM2wtMC40LTAuMWMtMC41LTAuMi0xMy4yLTUuOC0xMy4yLTE1LjlWOC4xaDI3LjJ2MTguNGMwLDkuNy0xMi42LDE1LjMtMTMuMiwxNS42TDIyLjQsNDIuM3oKCQkJIE0xMC44LDkuOXYxNi4zYzAsOC4xLDkuNywxMy4xLDExLjgsMTQuMWMyLTEsMTEuOC02LjEsMTEuOC0xMy43VjEwSDEwLjhDMTAuOCwxMCwxMC44LDkuOSwxMC44LDkuOXoiLz4KCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNMTYuNywxOC44bC0wLjItMC4xYzAsMC0xLjUtMC45LTIuMy0xLjVjLTAuNy0wLjUtMS4xLTEuMS0xLjItMS42YzAtMC4xLDAtMS42LDAtNC41di0wLjZoNy43VjExCgkJCWMwLDIuOSwwLDQuNCwwLDQuNWMwLDAuNi0wLjQsMS4xLTEuMSwxLjZjLTEsMC42LTIuMywxLjUtMi40LDEuNUwxNi43LDE4Ljh6IE0xNCwxMS41YzAsMS42LDAsMy43LDAsMy45YzAsMC4xLDAuMiwwLjUsMC42LDAuNwoJCQljMC42LDAuNCwxLjUsMSwyLDEuMmMwLjUtMC4yLDEuNS0wLjksMi4xLTEuMmMwLjItMC4xLDAuNi0wLjQsMC42LTAuNmMwLTAuMiwwLTIuMiwwLTMuOUgxNEwxNCwxMS41TDE0LDExLjV6Ii8+CgkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTI4LjcsMTguOGwtMC4yLTAuMWMwLDAtMS41LTAuOS0yLjMtMS41Yy0wLjctMC41LTEuMS0xLjEtMS4yLTEuNmMwLTAuMSwwLTEuNiwwLTQuNXYtMC42aDcuN1YxMQoJCQljMCwyLjksMCw0LjQsMCw0LjVjMCwwLjYtMC40LDEuMS0xLjEsMS42Yy0xLDAuNi0yLjMsMS41LTIuNCwxLjVMMjguNywxOC44eiBNMjYsMTEuNWMwLDEuNiwwLDMuNywwLDMuOWMwLDAuMSwwLjIsMC41LDAuNiwwLjcKCQkJYzAuNiwwLjQsMS41LDEsMiwxLjJjMC41LTAuMiwxLjUtMC45LDIuMS0xLjJjMC4yLTAuMSwwLjYtMC40LDAuNi0wLjZjMC0wLjIsMC0yLjIsMC0zLjlIMjZMMjYsMTEuNUwyNiwxMS41eiIvPgoJCTxyZWN0IHg9IjkuMyIgeT0iMTkuMSIgY2xhc3M9InN0MCIgd2lkdGg9IjI2LjUiIGhlaWdodD0iMS42Ii8+CgkJPGc+CgkJCTxnPgoJCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTIyLjQsMzUuMmMtMC41LDAtMC43LTAuNC0wLjktMC41Yy0wLjEtMC4xLTAuMi0wLjItMC40LTAuMmMtMC43LDAtMS4yLDAtMS44LDAuMWMtMC42LDAtMS4yLDAuMS0yLjIsMC4xCgkJCQkJcy0xLjcsMC0xLjcsMGgtMC43VjIyLjNoMC43YzAuNSwwLDEuMSwwLDIuMSwwYzAuNSwwLDEtMC4xLDEuNi0wLjFjMC40LDAsMC43LTAuMSwxLjEtMC4xYzAuOS0wLjEsMS42LDAuMSwxLjcsMC4xCgkJCQkJYzAuMiwwLDAuNCwwLjEsMC42LDAuMmMwLjEtMC4xLDAuNC0wLjEsMC42LTAuMmMwLDAsMC45LTAuMSwxLjctMC4xYzAuNCwwLDAuNywwLjEsMS4xLDAuMWMwLjYsMC4xLDEuMSwwLjEsMS42LDAuMQoJCQkJCWMxLDAsMS42LDAsMi4xLDBoMC43djEyLjRoLTAuN2MwLDAtMC43LDAtMS43LDBjLTEsMC0xLjYtMC4xLTIuMi0wLjFjLTAuNiwwLTEuMS0wLjEtMS44LTAuMWMtMC4yLDAtMC4yLDAtMC40LDAuMgoJCQkJCUMyMy4yLDM1LDIyLjksMzUuMiwyMi40LDM1LjJ6IE0yMS4yLDMzLjFjMC42LDAsMS4xLDAuMiwxLjQsMC41YzAuMi0wLjIsMC43LTAuNSwxLjQtMC41YzAuNywwLDEuNCwwLDIsMC4xCgkJCQkJYzAuNiwwLDEuMiwwLjEsMi4xLDAuMWMwLjQsMCwwLjYsMCwwLjksMHYtOS41Yy0wLjQsMC0wLjksMC0xLjQsMGMtMC41LDAtMS4xLTAuMS0xLjctMC4xYy0wLjQsMC0wLjctMC4xLTEuMS0wLjEKCQkJCQljLTAuNi0wLjEtMS4yLDAtMS4yLDBjLTAuMSwwLTAuMiwwLjEtMC4yLDAuMXMwLDAsMC4xLTAuMWwtMC43LTAuMWwtMC43LDAuMWMwLDAuMSwwLDAuMSwwLjEsMC4xYzAsMCwwLDAtMC4yLTAuMWwwLDAKCQkJCQljMCwwLTAuNi0wLjEtMS4yLDBjLTAuNCwwLTAuNywwLjEtMS4xLDAuMWMtMC42LDAuMS0xLjIsMC4xLTEuNywwLjFjLTAuNiwwLTEsMC0xLjQsMHY5LjVjMC4yLDAsMC42LDAsMC45LDAKCQkJCQljMC45LDAsMS41LTAuMSwyLjEtMC4xQzE5LjksMzMuMSwyMC40LDMzLjEsMjEuMiwzMy4xeiIvPgoJCQk8L2c+CgkJPC9nPgoJCTxyZWN0IHg9IjEzLjQiIHk9IjEyLjgiIGNsYXNzPSJzdDAiIHdpZHRoPSI2LjQiIGhlaWdodD0iMS4xIi8+CgkJPHJlY3QgeD0iMjEuOCIgeT0iMTkuNSIgY2xhc3M9InN0MCIgd2lkdGg9IjEuNSIgaGVpZ2h0PSIyMS44Ii8+CgkJPHBvbHlnb24gY2xhc3M9InN0MCIgcG9pbnRzPSIzMS40LDE1LjIgMjguNiwxMy40IDI2LDE1LjIgMjUuMywxNC4zIDI4LjYsMTIgMzIsMTQuMyAJCSIvPgoJCTxwYXRoIGNsYXNzPSJzdDIiIGQ9Ik0yOC41LDE1LjMiLz4KCQk8cmVjdCB4PSIxNy4yIiB5PSIyNS4xIiBjbGFzcz0ic3QwIiB3aWR0aD0iMy4yIiBoZWlnaHQ9IjEuMSIvPgoJCTxyZWN0IHg9IjE3LjIiIHk9IjI3LjciIGNsYXNzPSJzdDAiIHdpZHRoPSIzLjIiIGhlaWdodD0iMS4xIi8+CgkJPHJlY3QgeD0iMTcuMiIgeT0iMzAuMyIgY2xhc3M9InN0MCIgd2lkdGg9IjMuMiIgaGVpZ2h0PSIxLjEiLz4KCQk8cmVjdCB4PSIyNC4zIiB5PSIyNS4xIiBjbGFzcz0ic3QwIiB3aWR0aD0iMy4yIiBoZWlnaHQ9IjEuMSIvPgoJCTxyZWN0IHg9IjI0LjMiIHk9IjI3LjciIGNsYXNzPSJzdDAiIHdpZHRoPSIzLjIiIGhlaWdodD0iMS4xIi8+CgkJPHJlY3QgeD0iMjQuMyIgeT0iMzAuMyIgY2xhc3M9InN0MCIgd2lkdGg9IjMuMiIgaGVpZ2h0PSIxLjEiLz4KCTwvZz4KCTxnIGlkPSJMYXllcl8zIj4KCQk8Zz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTY1LjEsMjguN2MtMS4xLDAuNy0zLjEsMS4xLTQuMywxLjFjLTQuNywwLTcuOC0yLjctNy44LTcuMWMwLTIuMiwwLjktNCwyLjQtNS4zYzEuNS0xLjIsMy40LTEuOCw1LjYtMS44CgkJCQljMS44LDAsMy42LDAuNSw0LjUsMC45Yy0wLjIsMS0wLjQsMi0wLjQsMi45aC0wLjZ2LTEuNWMwLTAuNS0wLjctMC45LTEuNy0xLjJjLTAuNi0wLjItMS41LTAuNC0yLjItMC40Yy0zLjcsMC01LjYsMi43LTUuNiw2CgkJCQljMCwzLjksMi42LDYuNCw2LjUsNi40YzEuNSwwLDMuMS0wLjUsMy45LTEuMmwwLjEsMC4yTDY1LjEsMjguN3oiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTcwLDI5LjdjLTIuNCwwLTQuMi0yLTQuMi00LjVjMC0yLjksMS44LTUsNS01YzIuNCwwLDQuNCwyLDQuNCw0LjRjMCwyLjktMi4xLDUuMi01LjIsNS4yTDcwLDI5LjdMNzAsMjkuNwoJCQkJTDcwLDI5LjdMNzAsMjkuN3ogTTY3LjcsMjQuM2MwLDIuMSwwLjcsNC44LDMuMyw0LjhjMS44LDAsMi42LTEuOCwyLjYtMy42YzAtMi42LTEuMi00LjctMy4xLTQuN0M2OC4zLDIwLjcsNjcuNywyMi40LDY3LjcsMjQuMwoJCQkJeiIvPgoJCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNNzYuOCwyMi45YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNWMxLjEtMC4xLDIuMy0wLjUsMy4xLTAuOWwwLjEsMC4xdjJsMCwwCgkJCQlsMS4xLTEuMmMwLjUtMC42LDEuMS0wLjksMS42LTAuOWMwLjQsMCwwLjcsMC4xLDAuOSwwLjJsLTAuMSwyLjNoLTAuNGMtMC4yLTAuNS0wLjYtMC45LTEuNC0wLjljLTAuOSwwLTEuNywwLjktMS43LDIuMnYzLjQKCQkJCWMwLDAuNSwwLDEsMC4xLDEuMWMwLjEsMC4xLDAuNCwwLjIsMC45LDAuMmgxdjAuNmMtMSwwLTEuOCwwLTIuOCwwYy0xLDAtMS43LDAtMi40LDB2LTAuNmgwLjljMC40LDAsMC41LTAuMSwwLjYtMC4yCgkJCQljMC4xLTAuMiwwLjEtMC42LDAuMS0xLjFMNzYuOCwyMi45TDc2LjgsMjIuOUw3Ni44LDIyLjlMNzYuOCwyMi45TDc2LjgsMjIuOXoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTg1LjgsMjcuM2MwLDAuNSwwLDEsMC4xLDEuMWMwLjEsMC4xLDAuMiwwLjIsMC42LDAuMmgwLjl2MC42Yy0wLjcsMC0xLjUsMC0yLjQsMHMtMS43LDAtMi40LDB2LTAuNmgwLjkKCQkJCWMwLjQsMCwwLjUtMC4xLDAuNi0wLjJjMC4xLTAuMiwwLjEtMC42LDAuMS0xLjF2LTQuNGMwLTAuNiwwLTEuMS0wLjEtMS4yYy0wLjEtMC4xLTAuNC0wLjItMC43LTAuMmgtMC42di0wLjUKCQkJCWMxLjEtMC4xLDIuMy0wLjUsMy4xLTAuOWwwLjEsMC4xdjEuN2wwLDBsMS41LTEuMmMwLjUtMC40LDAuOS0wLjUsMS42LTAuNWMyLjMsMCwzLjIsMS4yLDMuMiwzLjR2My44YzAsMC41LDAsMSwwLjEsMS4xCgkJCQljMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC41LDAtMS4xLDAtMS42LDBjLTAuNSwwLTEuMSwwLTEuNiwwdi01LjVjMC0xLjYtMS0yLjQtMi4yLTIuNGMtMS4yLDAtMi40LDEtMi40LDEuOAoJCQkJTDg1LjgsMjcuM0w4NS44LDI3LjN6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xMDEuOSwyMy43YzAsMC4yLDAsMC41LDAsMC45Yy0xLjIsMC4yLTIuMywwLjItMy42LDAuMmgtMi44YzAsMi4zLDEuMSwzLjksMy42LDMuOWMxLDAsMS43LTAuNSwyLjQtMQoJCQkJbDAuMiwwLjJsLTAuNCwwLjZjLTAuMSwwLjItMS44LDEuMS0zLjIsMS4xYy0yLjksMC00LjUtMS44LTQuNS00LjVjMC0yLDAuNS0zLjEsMS41LTMuOGMwLjQtMC4yLDAuNy0wLjYsMS4yLTAuNwoJCQkJYzAuNy0wLjQsMS4yLTAuNSwyLTAuNUMxMDAuMywyMC4xLDEwMS45LDIxLjUsMTAxLjksMjMuN3ogTTk1LjUsMjMuOFYyNGMwLjksMC4xLDEuNywwLjEsMi40LDAuMWMwLjcsMCwxLjUsMCwyLjIsMAoJCQkJYzAtMS44LTAuNi0zLjMtMi4xLTMuM0M5Ni42LDIwLjcsOTUuNSwyMS44LDk1LjUsMjMuOHoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTEwMy43LDE3LjJjMC0wLjUsMC0wLjktMC41LTAuOWgtMS4xdi0wLjVjMS0wLjEsMi4xLTAuNCwzLjEtMC43bDAuMSwwLjF2MTIuMWMwLDAuNSwwLDEsMC4xLDEuMQoJCQkJYzAuMSwwLjEsMC4yLDAuMiwwLjYsMC4yaDAuOXYwLjZjLTAuNywwLTEuNSwwLTIuNCwwcy0xLjcsMC0yLjQsMHYtMC42aDAuOWMwLjQsMCwwLjUtMC4xLDAuNi0wLjJjMC4xLTAuMiwwLjEtMC42LDAuMS0xLjEKCQkJCUwxMDMuNywxNy4yTDEwMy43LDE3LjJ6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xMDguNywxNy4yYzAtMC41LDAtMC45LTAuNS0wLjlIMTA3di0wLjVjMS0wLjEsMi4xLTAuNCwzLjEtMC43bDAuMSwwLjF2MTIuMWMwLDAuNSwwLDEsMC4xLDEuMQoJCQkJYzAuMSwwLjEsMC4yLDAuMiwwLjYsMC4yaDAuOXYwLjZjLTAuNywwLTEuNSwwLTIuNCwwYy0xLDAtMS43LDAtMi40LDB2LTAuNmgwLjljMC40LDAsMC41LTAuMSwwLjYtMC4yYzAuMS0wLjIsMC4xLTAuNiwwLjEtMS4xCgkJCQlMMTA4LjcsMTcuMkwxMDguNywxNy4yeiIvPgoJCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNMTE3LjgsMTguMmMwLTAuNywwLTEuMi0wLjEtMS41cy0wLjQtMC4yLTAuOS0wLjJoLTF2LTAuNmMxLDAsMiwwLDIuOSwwYzAuOSwwLDEuOCwwLDIuOCwwdjAuNmgtMQoJCQkJYy0wLjUsMC0wLjcsMC4xLTAuOSwwLjJjLTAuMSwwLjItMC4xLDAuNy0wLjEsMS41djYuN2MwLDIuOCwxLjUsMy42LDQsMy42YzIuMSwwLDQtMC45LDQtNHYtNi4zYzAtMC43LDAtMS4yLTAuMS0xLjUKCQkJCWMtMC4xLTAuMi0wLjQtMC4yLTAuOS0wLjJoLTAuOXYtMC42YzAuNywwLDEuNiwwLDIuMywwYzAuNywwLDEuNSwwLDIuMywwdjAuNmgtMC45Yy0wLjUsMC0wLjcsMC4xLTAuOSwwLjIKCQkJCWMtMC4xLDAuMi0wLjEsMC43LTAuMSwxLjV2NS42YzAsNC4yLTEuNiw1LjktNS41LDUuOWMtMy4zLDAtNS4zLTEtNS4zLTQuNUwxMTcuOCwxOC4yTDExNy44LDE4LjJMMTE3LjgsMTguMnoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTEzMy4yLDI3LjNjMCwwLjUsMCwxLDAuMSwxLjFjMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC43LDAtMS41LDAtMi40LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42CgkJCQloMC45YzAuNCwwLDAuNS0wLjEsMC42LTAuMmMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMXYtNC40YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNQoJCQkJYzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjF2MS43bDAsMGwxLjUtMS4yYzAuNS0wLjQsMC45LTAuNSwxLjYtMC41YzIuMywwLDMuMiwxLjIsMy4yLDMuNHYzLjhjMCwwLjUsMCwxLDAuMSwxLjEKCQkJCWMwLjEsMC4xLDAuMiwwLjIsMC42LDAuMmgwLjl2MC42Yy0wLjUsMC0xLjEsMC0xLjYsMGMtMC41LDAtMS4xLDAtMS42LDB2LTUuNWMwLTEuNi0xLTIuNC0yLjItMi40Yy0xLjIsMC0yLjQsMS0yLjQsMS44CgkJCQlMMTMzLjIsMjcuM0wxMzMuMiwyNy4zTDEzMy4yLDI3LjNMMTMzLjIsMjcuM3oiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE0NC45LDI3LjNjMCwwLjUsMCwxLDAuMSwxLjFjMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC43LDAtMS41LDAtMi40LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42CgkJCQloMC45YzAuNCwwLDAuNS0wLjEsMC42LTAuMmMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMXYtNC40YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNQoJCQkJYzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjFMMTQ0LjksMjcuM0wxNDQuOSwyNy4zTDE0NC45LDI3LjNMMTQ0LjksMjcuM3ogTTE0NS4xLDE2LjljMCwwLjYtMC41LDEuMS0xLjEsMS4xCgkJCQljLTAuNiwwLTEuMS0wLjUtMS4xLTEuMXMwLjUtMS4xLDEuMS0xLjFDMTQ0LjYsMTUuOCwxNDUuMSwxNi4zLDE0NS4xLDE2Ljl6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xNTIuMywyNy4zYy0wLjQsMC43LTAuNSwxLjUtMC45LDIuMWgtMWwtMy40LThjLTAuMS0wLjItMC4yLTAuNi0wLjYtMC42aC0wLjZ2LTAuNWMwLjcsMCwxLjUsMCwyLjMsMAoJCQkJYzAuNywwLDEuNSwwLDIuMywwdjAuNWgtMWMtMC40LDAtMC41LDAuMS0wLjUsMC40YzAsMC4xLDAsMC40LDAuMSwwLjdsMi4zLDUuNmMwLjQtMC45LDAuOS0xLjgsMS4yLTIuN2wwLjktMi4xCgkJCQljMC4yLTAuNiwwLjQtMS4xLDAuNC0xLjVjMC0wLjQtMC4xLTAuNS0wLjUtMC41aC0wLjl2LTAuNWMwLjYsMCwxLjIsMCwxLjgsMGMwLjYsMCwxLjIsMCwxLjgsMHYwLjVoLTAuNmMtMC41LDAtMC45LDAuNy0xLjEsMS40CgkJCQlMMTUyLjMsMjcuM3oiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE2NC4xLDIzLjdjMCwwLjIsMCwwLjUsMCwwLjljLTEuMiwwLjItMi4zLDAuMi0zLjYsMC4yaC0yLjhjMCwyLjMsMS4xLDMuOSwzLjYsMy45YzEsMCwxLjctMC41LDIuNC0xCgkJCQlsMC4yLDAuMmwtMC40LDAuNmMtMC4xLDAuMi0xLjgsMS4xLTMuMiwxLjFjLTIuOSwwLTQuNS0xLjgtNC41LTQuNWMwLTIsMC41LTMuMSwxLjUtMy44YzAuNC0wLjIsMC43LTAuNiwxLjItMC43CgkJCQljMC43LTAuNCwxLjItMC41LDItMC41QzE2Mi41LDIwLjEsMTY0LjEsMjEuNSwxNjQuMSwyMy43eiBNMTU3LjYsMjMuOFYyNGMwLjksMC4xLDEuNywwLjEsMi40LDAuMXMxLjUsMCwyLjIsMAoJCQkJYzAtMS44LTAuNi0zLjMtMi4xLTMuM0MxNTguOCwyMC43LDE1Ny42LDIxLjgsMTU3LjYsMjMuOHoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE2Ni4zLDIyLjljMC0wLjYsMC0xLjEtMC4xLTEuMmMtMC4xLTAuMS0wLjQtMC4yLTAuNy0wLjJoLTAuN3YtMC41YzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjF2MmwwLDAKCQkJCWwxLjEtMS4yYzAuNS0wLjYsMS4xLTAuOSwxLjYtMC45YzAuNCwwLDAuNywwLjEsMC45LDAuMmwtMC4xLDIuM2gtMC40Yy0wLjItMC41LTAuNi0wLjktMS40LTAuOWMtMC45LDAtMS43LDAuOS0xLjcsMi4ydjMuNAoJCQkJYzAsMC41LDAsMSwwLjEsMS4xYzAuMSwwLjEsMC40LDAuMiwwLjksMC4yaDF2MC42Yy0xLDAtMS44LDAtMi44LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42aDAuOWMwLjQsMCwwLjUtMC4xLDAuNi0wLjIKCQkJCWMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMUwxNjYuMywyMi45TDE2Ni4zLDIyLjlMMTY2LjMsMjIuOUwxNjYuMywyMi45eiIvPgoJCQk8cGF0aCBjbGFzcz0ic3QwIiBkPSJNMTczLDI2LjV2MC45YzAsMS4yLDEuNCwxLjcsMi42LDEuN2MxLjIsMCwyLjMtMC43LDIuMy0xLjhjMC0wLjYtMC40LTEuMS0xLTEuM2MtMC45LTAuMi0yLTAuNS0yLjktMC43CgkJCQljLTEtMC40LTEuNy0xLTEuNy0yLjFjMC0yLjEsMS44LTIuOCwzLjctMi44YzEsMCwxLjcsMC4yLDIuNiwwLjVjMCwwLjctMC4xLDEuNS0wLjEsMi4yaC0wLjV2LTAuNWMwLTEtMS4xLTEuNi0yLjMtMS42CgkJCQljLTEuNywwLTIsMS0yLDEuNmMwLDAuOSwwLjYsMS40LDIuMSwxLjZjMi4zLDAuNCwzLjQsMSwzLjQsMi40YzAsMi4yLTIuMiwzLjMtNC4zLDMuM2MtMSwwLTEuOC0wLjEtMi43LTAuNQoJCQkJYzAuMi0wLjksMC4yLTEuOCwwLjItMi43aDAuNkwxNzMsMjYuNUwxNzMsMjYuNUwxNzMsMjYuNUwxNzMsMjYuNXoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE4My4yLDI3LjNjMCwwLjUsMCwxLDAuMSwxLjFjMC4xLDAuMSwwLjIsMC4yLDAuNiwwLjJoMC45djAuNmMtMC43LDAtMS41LDAtMi40LDBjLTEsMC0xLjcsMC0yLjQsMHYtMC42CgkJCQloMC45YzAuNCwwLDAuNS0wLjEsMC42LTAuMmMwLjEtMC4yLDAuMS0wLjYsMC4xLTEuMXYtNC40YzAtMC42LDAtMS4xLTAuMS0xLjJjLTAuMS0wLjEtMC40LTAuMi0wLjctMC4yaC0wLjd2LTAuNQoJCQkJYzEuMS0wLjEsMi4zLTAuNSwzLjEtMC45bDAuMSwwLjFMMTgzLjIsMjcuM0wxODMuMiwyNy4zTDE4My4yLDI3LjNMMTgzLjIsMjcuM3ogTTE4My40LDE2LjljMCwwLjYtMC41LDEuMS0xLjEsMS4xCgkJCQljLTAuNiwwLTEuMS0wLjUtMS4xLTEuMXMwLjUtMS4xLDEuMS0xLjFDMTgyLjgsMTUuOCwxODMuNCwxNi4zLDE4My40LDE2Ljl6Ii8+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xODQuNSwyMnYtMC40bDEuNS0wLjd2LTEuM2MwLTAuNSwwLTEtMC4xLTEuNmMwLjctMC4yLDEuNC0wLjUsMS43LTAuN2wwLjIsMC4yYy0wLjEsMC45LTAuMiwyLTAuMiwyLjhWMjEKCQkJCWwyLjctMC4xbC0wLjEsMS4xaC0yLjR2NS4yYzAsMC45LDAuMiwxLjQsMS4xLDEuNGMwLjUsMCwwLjktMC4yLDEuMS0wLjRsMC4yLDAuNGwtMSwxYy0wLjEsMC4yLTAuOSwwLjItMS4yLDAuMgoJCQkJYy0xLDAtMi0wLjUtMi0yLjF2LTUuN0wxODQuNSwyMnoiLz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTE5OC4zLDIyYzAuMS0wLjIsMC4xLTAuNCwwLjEtMC41YzAtMC40LTAuMi0wLjUtMC45LTAuNUgxOTd2LTAuNWMwLjYsMCwxLjIsMCwxLjgsMGMwLjYsMCwxLjIsMCwxLjgsMFYyMQoJCQkJaC0wLjVjLTAuNSwwLTAuOSwwLjYtMS42LDIuM2wtMy45LDkuMmMtMC42LDEuNS0xLjMsMi40LTIuOSwyLjRjLTAuNCwwLTAuNy0wLjEtMS0wLjJsMC41LTEuNWgwLjJjMC4yLDAuMiwwLjcsMC41LDEsMC41bDAsMAoJCQkJYzEuMS0wLjEsMS43LTEuNywyLjEtMi42bDAuNS0xLjJsLTMuMi04Yy0wLjQtMC43LTAuNi0xLTEtMWgtMC40di0wLjVjMC43LDAsMS41LDAsMi4zLDBjMC43LDAsMS41LDAsMi4zLDBWMjFoLTAuNwoJCQkJYy0wLjQsMC0wLjYsMC4xLTAuNiwwLjVjMCwwLjIsMCwwLjUsMC4xLDAuN2wyLjIsNS40TDE5OC4zLDIyeiIvPgoJCTwvZz4KCTwvZz4KPC9nPgo8L3N2Zz4K 400w" sizes=400w>
<source srcset="data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE5LjIuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxheWVyXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IgoJIHZpZXdCb3g9IjAgMCAxMjAgMTIwIiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCAxMjAgMTIwOyIgeG1sOnNwYWNlPSJwcmVzZXJ2ZSI+CjxnPgoJPHBhdGggZD0iTTQ1LjksOTEuN2M2LjgsNS4zLDEzLjUsOC4yLDEzLjgsOC4zbDAuNSwwLjJsMC41LTAuMmMwLjMtMC4xLDYuOS0zLDEzLjctOC4zYzkuMS03LjEsMTMuOS0xNS4yLDEzLjktMjMuNWwwLTM3LjZsLTU2LjQsMAoJCWwwLDM3QzMxLjksNzYuMSwzNi43LDg0LjUsNDUuOSw5MS43eiBNNTguOCw4MC4xYy0wLjMtMC4xLTAuNi0wLjItMS0wLjJjMCwwLDAsMC0wLjEsMGMtMS40LDAtMi40LDAuMS0zLjUsMC4yCgkJYy0xLjEsMC4xLTIuMiwwLjItMy44LDAuMmMtMC43LDAtMS40LDAtMS45LDBWNjIuN2MwLjEsMCwwLjIsMCwwLjIsMGMwLjcsMCwxLjUsMCwyLjYsMGMwLjksMCwxLjktMC4xLDMtMC4yCgkJYzAuNi0wLjEsMS4zLTAuMSwyLTAuMmMxLjItMC4xLDIuMiwwLjEsMi4yLDAuMWMwLjEsMCwwLjIsMC4xLDAuMywwLjFDNTguOCw2Mi40LDU4LjgsODAuMSw1OC44LDgwLjF6IE04NS42LDY4LjIKCQljMCwxNS40LTE4LjksMjUuOC0yNC41LDI4LjZ2LTE0YzAuMS0wLjEsMC4yLTAuMiwwLjMtMC4zYzAuMy0wLjMsMC41LTAuNSwxLTAuNWMxLjMsMCwyLjMsMC4xLDMuNCwwLjJjMS4xLDAuMSwyLjIsMC4yLDMuOSwwLjIKCQljMS43LDAsMywwLDMsMGwwLjksMGwwLTUuMmgyLjJ2LTIuOGgtMi4ybDAtNS4xaDIuMnYtMi44aC0yLjJsMC01LjdsLTEsMGMtMC40LDAtMC44LDAtMS4zLDBjLTAuNywwLTEuNSwwLTIuNSwwCgkJYy0wLjgsMC0xLjctMC4xLTIuOC0wLjJjLTAuNi0wLjEtMS4zLTAuMS0yLTAuMmMtMS41LTAuMi0yLjgsMC4xLTIuOSwwLjFjMCwwLDAsMC0wLjEsMFY1NWgyNC41TDg1LjYsNjguMnogTTYxLjEsODAuMlY2Mi41CgkJYzAuMS0wLjEsMC4zLTAuMSwwLjUtMC4yYzAsMCwxLjEtMC4yLDIuMy0wLjFjMC43LDAuMSwxLjQsMC4xLDIsMC4yYzEuMSwwLjEsMi4xLDAuMiwzLDAuMmMwLjksMCwxLjYsMCwyLjMsMGMwLjIsMCwwLjQsMCwwLjUsMAoJCWwwLDE3LjdjLTAuNSwwLTEuMiwwLTEuOSwwYy0xLjYsMC0yLjctMC4xLTMuOC0wLjJjLTEuMS0wLjEtMi4xLTAuMi0zLjUtMC4yQzYxLjksODAsNjEuNSw4MC4xLDYxLjEsODAuMnogTTM0LjYsMzMuM2w1MSwwbDAsMTkuMQoJCWgtNTFMMzQuNiwzMy4zeiBNMzQuNiw1NWgyNC4zdjUuM2MtMC40LTAuMS0xLjUtMC4yLTIuNy0wLjFjLTAuNywwLjEtMS40LDAuMS0yLDAuMmMtMS4xLDAuMS0yLDAuMi0yLjgsMC4yYy0xLDAtMS44LDAtMi41LDAKCQljLTAuNSwwLTAuOSwwLTEuMywwbC0xLDB2NS4xaC0yLjR2Mi44aDIuNHY1LjFoLTIuNHYyLjhoMi40djUuN2wwLjksMGMwLjEsMCwxLjQsMC4xLDMsMGMxLjcsMCwyLjktMC4xLDMuOS0wLjIKCQljMS4xLTAuMSwyLTAuMiwzLjMtMC4yYzAuNSwwLDAuNywwLjIsMC45LDAuNWMwLDAsMC4xLDAuMSwwLjEsMC4xdjE0LjFDNTMsOTMuOCwzNC42LDgzLjUsMzQuNiw2Ny42TDM0LjYsNTV6Ii8+Cgk8cGF0aCBkPSJNNTUuOSwzNS4ySDQxLjZ2MWMwLDMuMSwwLDguNCwwLDguNmMwLjEsMS4xLDAuOCwyLjEsMi4yLDNjMS43LDEuMiw0LjMsMi43LDQuNCwyLjdsMC41LDAuM2wwLjUtMC4zCgkJYzAuMS0wLjEsMi45LTEuNiw0LjYtMi43YzItMS4zLDIuMS0yLjYsMi4xLTNjMC0wLjIsMC00LjgsMC04LjZWMzUuMnogTTQzLjYsMzcuMmgxMC4yYzAsMC44LDAsMS43LDAsMi42SDQzLjYKCQlDNDMuNiwzOC45LDQzLjYsMzgsNDMuNiwzNy4yeiBNNTMuOSw0NC43YzAsMC41LTAuNywxLTEuMiwxLjRjLTEuMiwwLjgtMy4xLDEuOS00LDIuNGMtMC44LTAuNS0yLjYtMS41LTMuOC0yLjQKCQljLTAuOC0wLjUtMS4zLTEuMS0xLjMtMS41YzAtMC4xLDAtMS4zLDAtMi45aDEwLjJDNTMuOSw0My4zLDUzLjksNDQuNSw1My45LDQ0Ljd6Ii8+Cgk8cGF0aCBkPSJNODAuMSwzNC44SDY1LjZ2MWMwLDMuOSwwLDguNiwwLDguOWMwLjEsMS4xLDAuOCwyLjEsMi4yLDNjMS44LDEuMiw0LjQsMi43LDQuNSwyLjhsMC41LDAuM2wwLjUtMC4zCgkJYzAuMS0wLjEsMy0xLjYsNC43LTIuOGMyLjEtMS40LDIuMS0yLjYsMi4xLTNjMC0wLjIsMC00LjksMC04LjhMODAuMSwzNC44eiBNNzguMSw0NC41YzAsMC41LTAuNywxLjEtMS4yLDEuNAoJCWMtMS4zLDAuOC0zLjIsMS45LTQuMSwyLjRjLTAuOS0wLjUtMi42LTEuNi0zLjktMi40Yy0wLjQtMC4zLTEuMy0wLjktMS4zLTEuNWMwLTAuMSwwLTAuMiwwLTAuNWw1LTMuNmw1LjUsMy43CgkJQzc4LjEsNDQuMyw3OC4xLDQ0LjUsNzguMSw0NC41eiBNNzIuNiwzNy44bC01LDMuNmMwLTEuNCwwLTMuMiwwLTQuN2gxMC42YzAsMS42LDAsMy4zLDAsNC44TDcyLjYsMzcuOHoiLz4KCTxwYXRoIGQ9Ik02My43LDY2LjVsMSwwLjFjMC42LDAuMSwxLjIsMC4xLDEuNywwLjFsMi45LDAuMWwwLTJsLTIuOS0wLjFjLTAuNCwwLTAuOS0wLjEtMS41LTAuMWwtMS0wLjFjLTAuMywwLTAuNSwwLTAuNywwbDAuMSwyCgkJQzYzLjQsNjYuNCw2My42LDY2LjQsNjMuNyw2Ni41eiIvPgoJPHBhdGggZD0iTTY2LjQsNzAuMmMtMC40LDAtMC45LTAuMS0xLjUtMC4xYy0wLjMsMC0wLjctMC4xLTEtMC4xYy0wLjMsMC0wLjUsMC0wLjcsMGwwLDJjMC4xLDAsMC4zLDAsMC41LDBjMC40LDAsMC43LDAuMSwxLDAuMQoJCWMwLjYsMC4xLDEuMSwwLjEsMS42LDAuMWMwLjYsMCwxLjUsMCwyLjIsMGwwLjgsMGwwLTJsLTAuOCwwQzY3LjksNzAuMyw2Nyw3MC4yLDY2LjQsNzAuMnoiLz4KCTxwYXRoIGQ9Ik02Ni40LDc1LjhjLTAuNCwwLTAuOS0wLjEtMS41LTAuMWMtMC4zLDAtMC43LTAuMS0xLTAuMWMtMC4zLDAtMC41LDAtMC43LDBsMC4xLDJjMC4xLDAsMC4zLDAsMC41LDBjMC40LDAsMC43LDAuMSwxLDAuMQoJCWMwLjYsMC4xLDEuMiwwLjEsMS42LDAuMWwwLjgsMGMwLjgsMCwxLjgsMCwyLjEsMC4xbDAtMmMtMC4zLDAtMS4zLDAtMi4xLTAuMUw2Ni40LDc1Ljh6Ii8+Cgk8cGF0aCBkPSJNNTMuOCw2Ni43YzAuNSwwLDEtMC4xLDEuNy0wLjFjMC4zLDAsMC42LTAuMSwxLTAuMWMwLjIsMCwwLjMsMCwwLjUsMGwwLTJjLTAuMiwwLTAuNCwwLTAuNywwYy0wLjQsMC0wLjcsMC4xLTEsMC4xCgkJYy0wLjYsMC4xLTEuMSwwLjEtMS41LDAuMWwtMi45LDAuMWwwLDJMNTMuOCw2Ni43eiIvPgoJPHBhdGggZD0iTTU1LjMsNzAuMWMtMC42LDAuMS0xLjEsMC4xLTEuNSwwLjFsLTIuOSwwLjFsMCwybDIuOS0wLjFjMC41LDAsMS0wLjEsMS43LTAuMWwwLjktMC4xYzAuMiwwLDAuMywwLDAuNCwwbDAuMS0yCgkJYy0wLjIsMC0wLjQsMC0wLjcsMEw1NS4zLDcwLjF6Ii8+Cgk8cGF0aCBkPSJNNTUuMyw3NS42Yy0wLjYsMC4xLTEuMSwwLjEtMS42LDAuMWMtMC45LDAtMi40LDAuMS0yLjgsMC4xbDAsMmMwLjQsMCwxLjktMC4xLDIuOS0wLjFjMC41LDAsMS0wLjEsMS43LTAuMWwxLTAuMQoJCWMwLjEsMCwwLjMsMCwwLjQsMGwwLTJjLTAuMiwwLTAuNCwwLTAuNywwTDU1LjMsNzUuNnoiLz4KCTxwYXRoIGQ9Ik05LjMsNjAuM2MyLjUsMCw0LTEuNiw0LjItNC40YzAtMC41LDAtMS4yLTAuMS0xLjlsMC0wLjJsLTIuMSwwbDAuMiwwLjRjMC4xLDAuMywwLjIsMSwwLjIsMS42Yy0wLjEsMS40LTEuMiwyLjMtMi43LDIuMgoJCWMtMS42LTAuMS0yLjUtMS4xLTIuNS0yLjVjMC0wLjUsMC4xLTEsMC41LTEuNWwwLjMtMC40bC0yLTAuM2wtMC4xLDAuMmMtMC4zLDAuNi0wLjQsMS4zLTAuNSwxLjljLTAuMiwyLjksMS40LDQuOCw0LjMsNC45CgkJQzkuMSw2MC4zLDkuMiw2MC4zLDkuMyw2MC4zeiIvPgoJPHBhdGggZD0iTTkuNyw1MC41YzAuNiwwLjIsMS4xLDAuMiwxLjYsMC4yYzEuOSwwLDMuMy0xLjEsMy45LTNjMC4zLTEuMiwwLjItMi4yLTAuMy0zLjJjLTAuNi0xLTEuNi0xLjctMi45LTIuMQoJCWMtMC41LTAuMi0xLjEtMC4yLTEuNS0wLjJjLTEuOSwwLTMuMywxLjEtMy45LDNDNS44LDQ3LjcsNyw0OS43LDkuNyw1MC41eiBNOC4zLDQ1LjhjMC4zLTEuMSwxLjItMS4zLDEuOS0xLjMKCQljMC4zLDAsMC43LDAuMSwxLjEsMC4yYzAuOSwwLjMsMS42LDAuNywxLjksMS4zYzAuMiwwLjQsMC4zLDAuOCwwLjEsMS4zYy0wLjMsMS4xLTEuMiwxLjMtMS45LDEuM2MtMC40LDAtMC43LTAuMS0xLjEtMC4yCgkJQzkuMSw0OCw3LjksNDcuMSw4LjMsNDUuOHoiLz4KCTxwYXRoIGQ9Ik0xOC4zLDM5LjNsLTIuOS0xLjZsMC4xLTAuMmMwLjMtMC41LDAuNS0wLjUsMC45LTAuNGwyLjksMC40bDEuMS0yLjFsLTMuOC0wLjRjLTAuMywwLTAuNSwwLTAuNywwYzAtMC44LTAuNC0xLjYtMS4yLTIKCQljLTAuNC0wLjItMC44LTAuMy0xLjItMC4zYy0xLjQsMC0yLjEsMS40LTIuOCwyLjdsLTEsMS45bDcuNiw0LjFMMTguMywzOS4zeiBNMTMuOCwzNi42bC0wLjEsMC4ybC0xLjQtMC43bDAuMS0wLjIKCQljMC4zLTAuNiwwLjctMS4yLDEuMy0wLjljMC4yLDAuMSwwLjMsMC4yLDAuNCwwLjRDMTQuMiwzNS44LDE0LDM2LjMsMTMuOCwzNi42eiIvPgoJPHBvbHlnb24gcG9pbnRzPSIyMywzMS43IDE5LjEsMjguNSAyNC43LDI5LjggMjYuMiwyNy45IDE5LjYsMjIuNCAxOC4yLDI0IDIyLjIsMjcuMyAxNi42LDI2IDE1LDI3LjkgMjEuNywzMy40IAkiLz4KCTxwb2x5Z29uIHBvaW50cz0iMzIuNywyMi40IDMxLjUsMjEgMjkuMSwyMi45IDI4LDIxLjYgMzAuMiwxOS44IDI5LDE4LjQgMjYuOCwyMC4yIDI2LDE5LjEgMjguNCwxNy4yIDI3LjIsMTUuNyAyMy4xLDE5IDI4LjUsMjUuNyAKCQkJIi8+Cgk8cG9seWdvbiBwb2ludHM9IjM5LjgsMTguMSAzOC45LDE2LjUgMzYuMiwxOCAzMi45LDEyLjEgMzEsMTMuMiAzNS4yLDIwLjcgCSIvPgoJPHBvbHlnb24gcG9pbnRzPSI0Ny4xLDE1LjMgNDYuNCwxMy42IDQzLjYsMTQuNiA0MS4zLDguMyAzOS4yLDkgNDIuMSwxNy4xIAkiLz4KCTxwYXRoIGQ9Ik01NS44LDEzLjljMC4yLDAsMC4zLDAsMC41LDBjMS4yLTAuMSwyLjEtMC41LDIuNi0xLjFjMC41LTAuNiwwLjctMS41LDAuNi0yLjVsLTAuNC01LjRMNTYuOSw1bDAuNCw0LjkKCQljMC4xLDEuNy0wLjYsMi0xLjIsMi4xYy0wLjEsMC0wLjEsMC0wLjIsMGMtMC41LDAtMS4yLTAuMi0xLjQtMS45bC0wLjQtNC45bC0yLjIsMC4ybDAuNCw1LjRDNTIuNywxMi44LDUzLjgsMTMuOSw1NS44LDEzLjl6Ii8+Cgk8cG9seWdvbiBwb2ludHM9IjY1LjcsOC44IDY3LjUsMTQuMSA3MCwxNC41IDcxLjEsNS45IDY5LDUuNiA2OC4zLDEwLjcgNjYuNSw1LjMgNjQuMSw1IDYyLjksMTMuNSA2NSwxMy44IAkiLz4KCTxwb2x5Z29uIHBvaW50cz0iNzcuNiw3LjggNzUuNSw3LjEgNzIuOSwxNS4zIDc1LDE2IAkiLz4KCTxwb2x5Z29uIHBvaW50cz0iODguNCwxMi42IDg2LjUsMTEuNiA4Mi4zLDE1LjkgODMuMiwxMCA4MS4xLDguOSA3OS44LDE4IDgyLDE5IAkiLz4KCTxwb2x5Z29uIHBvaW50cz0iOTAuNiwyNSA5MS43LDIzLjUgODkuMywyMS42IDkwLjMsMjAuMyA5Mi41LDIyIDkzLjcsMjAuNSA5MS41LDE4LjggOTIuMywxNy43IDk0LjcsMTkuNiA5NS45LDE4LjEgOTEuNywxNC45IAoJCTg2LjQsMjEuNyAJIi8+Cgk8cGF0aCBkPSJNOTQuMiwyOC42bDIuNS0yLjJsMC4xLDAuMmMwLjMsMC40LDAuMywwLjUsMC4xLDFsLTEuNCwyLjZsMS42LDEuOGwxLjctMy40YzAuMS0wLjEsMC4yLTAuNCwwLjItMC43CgkJYzAuNywwLjMsMS42LDAuMiwyLjMtMC40YzEuOS0xLjYsMC40LTMuMy0wLjctNC42bC0xLjQtMS42TDkyLjcsMjdMOTQuMiwyOC42eiBNOTkuMywyNC4xbDAuMSwwLjJjMC4zLDAuNCwwLjYsMC44LDAuNiwxLjEKCQljMCwwLjItMC4xLDAuMy0wLjMsMC40Qzk5LjYsMjYsOTkuNCwyNiw5OS4zLDI2Yy0wLjQsMC0wLjgtMC40LTEtMC43bC0wLjEtMC4yTDk5LjMsMjQuMXoiLz4KCTxwYXRoIGQ9Ik0xMDIuMiwzNS41Yy0wLjQsMC4zLTEtMC4yLTEuMi0wLjZjLTAuMy0wLjUtMC40LTEtMC40LTEuN2wwLTAuNWwtMiwwLjlsMC4xLDAuNGMwLjIsMC43LDAuMywxLjEsMC43LDEuOAoJCWMwLjcsMS4yLDEuNiwxLjgsMi42LDEuOGMwLjUsMCwwLjktMC4xLDEuNC0wLjRjMS41LTAuOSwxLjMtMi4zLDEuMS0zLjRjLTAuMS0wLjctMC4yLTEuMiwwLjEtMS40YzAuMi0wLjEsMC4zLTAuMSwwLjQtMC4xCgkJYzAuNCwwLDAuNywwLjQsMC44LDAuNWMwLjMsMC40LDAuNCwwLjksMC41LDEuNGwwLjEsMC40bDEuOC0wLjhsMC0wLjJjLTAuMS0wLjYtMC40LTEuMi0wLjgtMS45Yy0wLjktMS42LTIuNS0yLjEtMy45LTEuMgoJCWMtMS41LDAuOS0xLjMsMi4zLTEuMSwzLjNDMTAyLjUsMzQuOCwxMDIuNiwzNS4zLDEwMi4yLDM1LjV6Ii8+Cgk8cmVjdCB4PSIxMDIuNSIgeT0iMzkuNiIgdHJhbnNmb3JtPSJtYXRyaXgoMC45MjU5IC0wLjM3NzcgMC4zNzc3IDAuOTI1OSAtNy40NTU5IDQzLjM1NjgpIiB3aWR0aD0iOC42IiBoZWlnaHQ9IjIuMiIvPgoJPHBvbHlnb24gcG9pbnRzPSIxMTEuMiw0Ni4yIDEwNC43LDQ3LjggMTA1LjIsNDkuOSAxMTEuOCw0OC4zIDExMi4zLDUwLjUgMTE0LjEsNTAuMSAxMTIuNSw0My41IDExMC43LDQ0IAkiLz4KCTxwb2x5Z29uIHBvaW50cz0iMTA2LjIsNTYuNiAxMDYuMyw1OC44IDEwOS43LDU4LjYgMTE1LjEsNjEuNCAxMTUsNTkgMTExLjgsNTcuNSAxMTQuOCw1NS42IDExNC42LDUzIDEwOS42LDU2LjUgCSIvPgoJPHBvbHlnb24gcG9pbnRzPSIxMSw3Ni40IDEyLjgsNzUuOSAxMi4xLDczLjMgMTMuNCw3Mi45IDE0LjIsNzUuNyAxNiw3NS4xIDE0LjYsNzAuMyA2LjMsNzIuNyA2LjksNzQuOCAxMC4zLDczLjggCSIvPgoJPHBhdGggZD0iTTE4LjcsODMuNmMwLjMtMSwwLjItMi4xLTAuMy0zLjJjLTAuOC0xLjYtMi0yLjQtMy42LTIuNGMtMC43LDAtMS41LDAuMi0yLjMsMC42Yy0xLjMsMC42LTIuMSwxLjUtMi41LDIuNgoJCWMtMC4zLDEtMC4yLDIuMSwwLjMsMy4yYzAuOCwxLjUsMiwyLjQsMy42LDIuNGMwLjcsMCwxLjUtMC4yLDIuMy0wLjZDMTcuNSw4NS41LDE4LjQsODQuNywxOC43LDgzLjZ6IE0xNS4zLDg0LjEKCQljLTAuNSwwLjMtMS4xLDAuNC0xLjYsMC40Yy0wLjgsMC0xLjMtMC4zLTEuNi0xYy0wLjItMC41LTAuMy0wLjktMC4xLTEuM2MwLjItMC42LDAuOC0xLjIsMS42LTEuNmMwLjUtMC4yLDEtMC40LDEuNS0wLjQKCQljMC44LDAsMS4zLDAuMywxLjYsMWMwLjIsMC41LDAuMywwLjksMC4xLDEuM0MxNi42LDgzLjEsMTYuMSw4My43LDE1LjMsODQuMXoiLz4KCTxwYXRoIGQ9Ik0yNC43LDkwLjNsLTMuOSwzLjFjLTEsMC44LTEuOCwwLjktMi40LDAuMmMtMC4yLTAuMy0wLjMtMC42LTAuMy0wLjljMC4xLTAuNCwwLjQtMC45LDEtMS40bDMuOS0zLjFsLTEuNC0xLjdsLTQuMywzLjQKCQljLTAuOSwwLjctMS40LDEuNS0xLjQsMi40YzAsMC44LDAuMywxLjYsMSwyLjVjMC44LDEuMSwxLjcsMS42LDIuNywxLjZjMC43LDAsMS40LTAuMywyLjEtMC45TDI2LDkyTDI0LjcsOTAuM3oiLz4KCTxwb2x5Z29uIHBvaW50cz0iMzIsOTcuMyAyOC44LDEwMS4zIDMwLDk1LjcgMjguMSw5NC4xIDIyLjYsMTAwLjggMjQuMywxMDIuMiAyNy41LDk4LjIgMjYuMywxMDMuOCAyOC4yLDEwNS40IDMzLjcsOTguNyAJIi8+Cgk8cGF0aCBkPSJNMzguNCwxMDEuNGwtMi4yLTEuMWwtMy45LDcuN2wyLjIsMS4xYzEuMSwwLjUsMiwwLjgsMi45LDAuOGMxLjUsMCwyLjctMC44LDMuNS0yLjRDNDIuMiwxMDQuOSw0MS4zLDEwMi45LDM4LjQsMTAxLjR6CgkJIE0zOC44LDEwNi41Yy0wLjQsMC45LTEuMiwxLjQtMi4xLDEuNGMtMC40LDAtMC43LTAuMS0xLjEtMC4zbC0wLjUtMC4zbDIuMi00LjRsMC41LDAuM2MwLjYsMC4zLDEsMC44LDEuMiwxLjMKCQlDMzkuMiwxMDUuMiwzOS4xLDEwNS44LDM4LjgsMTA2LjV6Ii8+Cgk8cG9seWdvbiBwb2ludHM9IjQzLjIsMTEyLjggNDguMywxMTQuMiA0OC44LDExMi40IDQ1LjgsMTExLjYgNDYuMiwxMTAgNDksMTEwLjcgNDkuNSwxMDguOSA0Ni43LDEwOC4yIDQ3LDEwNi44IDUwLjEsMTA3LjYgCgkJNTAuNSwxMDUuOCA0NS40LDEwNC41IAkiLz4KCTxwYXRoIGQ9Ik01NS45LDEwNi41bC0yLjUtMC4xbC0wLjUsOC42bDIuNSwwLjFjMC4yLDAsMC40LDAsMC42LDBjMi45LDAsNC40LTEuNCw0LjYtNC4xYzAuMS0xLjMtMC4yLTIuNC0wLjktMy4yCgkJQzU4LjksMTA3LjEsNTcuNywxMDYuNiw1NS45LDEwNi41eiBNNTUuOCwxMTMuM2wtMC42LDBsMC4zLTQuOWwwLjYsMGMxLjIsMC4xLDIuMywxLDIuMiwyLjZDNTguMiwxMTIuNiw1NywxMTMuMyw1NS44LDExMy4zeiIvPgoJPHBhdGggZD0iTTY4LjIsMTA1LjhsLTEuNSw5LjFsMi4zLTAuNWwwLjMtMS45bDIuOC0wLjZsMS4xLDEuN2wyLjQtMC41bC01LjEtNy43TDY4LjIsMTA1Ljh6IE02OS42LDExMC42bDAuMy0yLjNsMS4yLDEuOQoJCUw2OS42LDExMC42eiIvPgoJPHBvbHlnb24gcG9pbnRzPSI3Ni40LDExMC4zIDc3LjIsMTEyLjQgNzkuMiwxMTEuNiA3OC41LDEwOS42IAkiLz4KCTxwYXRoIGQ9Ik04MC42LDEwMS42bC0yLjIsMS4ybDQuMSw3LjZsMi4yLTEuMmMyLjktMS42LDMuNi0zLjYsMi4zLTYuMUM4NS43LDEwMC41LDgzLjUsMTAwLDgwLjYsMTAxLjZ6IE04NC4xLDEwNy40bC0wLjUsMC4zCgkJbC0yLjMtNC4zbDAuNS0wLjNjMC4zLTAuMiwwLjctMC4zLDEuMS0wLjNjMC45LDAsMS42LDAuNSwyLjEsMS4zQzg1LjgsMTA1LjUsODUuMSwxMDYuOCw4NC4xLDEwNy40eiIvPgoJPHBvbHlnb24gcG9pbnRzPSI4OC43LDEwNC4zIDg5LjksMTA2LjEgOTEuNywxMDQuOCA5MC41LDEwMyAJIi8+Cgk8cG9seWdvbiBwb2ludHM9IjkyLjEsOTMuMiA5MS43LDk2LjYgOTMuNiw5Ni45IDkzLjgsOTUuNSA5Ny45LDk5LjYgOTkuNSw5OC4xIDkzLjQsOTEuOSAJIi8+Cgk8cGF0aCBkPSJNMTAzLjgsODguN2MtMC43LTAuNS0xLjUtMC42LTIuNC0wLjNjMC0wLjgtMC40LTEuNi0xLTJjLTEuMS0wLjgtMi41LTAuMy0zLjYsMS4xYy0xLDEuNC0wLjksMi44LDAuMywzLjcKCQljMC42LDAuNCwxLjQsMC41LDIuMSwwLjNjLTAuMSwwLjksMC4yLDEuNiwwLjksMi4xYzAuNSwwLjQsMSwwLjUsMS41LDAuNWMwLjksMCwxLjctMC41LDIuNS0xLjVjMC42LTAuOCwwLjktMS42LDAuOC0yLjQKCQlDMTA0LjgsODkuNywxMDQuNCw4OS4yLDEwMy44LDg4Ljd6IE05OS43LDg5LjNjLTAuMywwLjItMC42LDAuNC0wLjksMC40Yy0wLjEsMC0wLjMsMC0wLjQtMC4xYy0wLjQtMC4zLTAuMi0wLjgsMC0xLjEKCQljMC4xLTAuMiwwLjMtMC4zLDAuNS0wLjNjMC4xLDAsMC4zLDAuMSwwLjUsMC4yQzk5LjcsODguNiw5OS44LDg5LDk5LjcsODkuM3ogTTEwMi43LDkxLjdjLTAuMywwLjQtMC44LDAuNS0xLjIsMC4yCgkJYy0wLjMtMC4zLTAuNS0wLjYtMC40LTEuMmMwLjYtMC40LDEuMS0wLjUsMS41LTAuMmMwLjIsMC4xLDAuMywwLjMsMC4zLDAuNUMxMDIuOSw5MS4yLDEwMi44LDkxLjUsMTAyLjcsOTEuN3oiLz4KCTxwYXRoIGQ9Ik0xMDcuNyw4MC4zYy0xLjUtMC43LTIuOS0wLjMtMy42LDEuMWMtMC4yLDAuNC0wLjMsMC44LTAuMiwxLjJjLTAuNS0wLjQtMS0xLjEtMC42LTJjMC4yLTAuNCwwLjUtMC43LDAuOC0xbDAuMy0wLjIKCQlsLTEuNi0xLjFsLTAuMiwwLjJjLTAuNCwwLjQtMC44LDAuOS0xLjEsMS41Yy0wLjQsMC45LTAuNSwxLjctMC4yLDIuNWMwLjUsMS40LDEuOSwyLjQsMywyLjljMSwwLjUsMS44LDAuNywyLjUsMC43CgkJYzEuMSwwLDEuOS0wLjUsMi41LTEuNkMxMTAuMSw4Mi43LDEwOS41LDgxLjIsMTA3LjcsODAuM3ogTTEwNy43LDgzLjVjLTAuMSwwLjMtMC40LDAuNC0wLjcsMC40Yy0wLjIsMC0wLjUtMC4xLTAuNy0wLjIKCQljLTAuNC0wLjItMC43LTAuNS0wLjgtMC43Yy0wLjEtMC4yLDAtMC4zLDAtMC41YzAuMS0wLjMsMC4zLTAuNCwwLjctMC40YzAuMiwwLDAuNSwwLjEsMC44LDAuMmMwLjQsMC4yLDAuNiwwLjQsMC43LDAuNwoJCUMxMDcuOCw4My4yLDEwNy43LDgzLjQsMTA3LjcsODMuNXoiLz4KCTxwYXRoIGQ9Ik0xMTAuOCw3MS41Yy0xLjctMC41LTMuMSwwLjMtMy42LDEuOWMwLDAuMi0wLjEsMC4zLTAuMSwwLjRsLTEtMC4zbDAuOS0zbC0xLjctMC41bC0xLjUsNWw0LjgsMS41bDAtMC40CgkJYzAtMC41LDAuMS0wLjksMC4yLTEuNGMwLjEtMC40LDAuNi0xLjQsMS41LTEuMmMxLDAuMywwLjgsMS40LDAuNywxLjdjLTAuMiwwLjYtMC41LDEtMC44LDEuNGwtMC4zLDAuM2wyLDAuN2wwLjEtMC4yCgkJYzAuMy0wLjYsMC42LTEuMywwLjgtMS44YzAuMy0xLDAuMy0xLjktMC4xLTIuN0MxMTIuMyw3Mi4zLDExMS42LDcxLjgsMTEwLjgsNzEuNXoiLz4KCTxwYXRoIGQ9Ik0xNi4zLDYwYzAsMjQsMTkuNSw0My42LDQzLjUsNDMuNmMyNCwwLDQzLjUtMTkuNSw0My41LTQzLjZjMC0yNC0xOS41LTQzLjUtNDMuNS00My41QzM1LjksMTYuNCwxNi4zLDM2LDE2LjMsNjB6CgkJIE01OS45LDE3LjljMjMuMiwwLDQyLDE4LjksNDIsNDJjMCwyMy4yLTE4LjksNDItNDIsNDJjLTIzLjIsMC00Mi0xOC45LTQyLTQyQzE3LjgsMzYuOCwzNi43LDE3LjksNTkuOSwxNy45eiIvPgoJPHBhdGggZD0iTTU5LjksMC40QzI3LDAuNCwwLjMsMjcuMSwwLjMsNjBjMCwzMi44LDI2LjcsNTkuNiw1OS42LDU5LjZjMzIuOCwwLDU5LjYtMjYuNyw1OS42LTU5LjZDMTE5LjQsMjcuMSw5Mi43LDAuNCw1OS45LDAuNHoKCQkgTTU5LjksMTE4Yy0zMiwwLTU4LTI2LTU4LTU4YzAtMzIsMjYtNTgsNTgtNThjMzIsMCw1OCwyNiw1OCw1OEMxMTcuOSw5Miw5MS45LDExOCw1OS45LDExOHoiLz4KPC9nPgo8L3N2Zz4K 2x">
</picture>
</a></div>
</div>
</div>
</header>
<main>
<div id=content>
<div id=abs-outer>
<div class=leftcolumn>
<div class=subheader>
<h1>Computer Science &gt; Software Engineering</h1>
</div>
<div class="header-breadcrumbs-mobile sf-hidden">
(cs)
</div>
<style>#abs{font-family:"Lucida Grande",Helvetica,Arial,sans-serif!important}#abs h1.title{display:block;font-size:1.8em!important;font-weight:700;margin-block-end:12px;margin-block-start:12px;margin-bottom:12px;margin-inline-end:0px;margin-inline-start:20px;margin-left:20px;margin-right:0px;margin-top:12px}#abs div.authors{font-size:1.2em;line-height:24px;margin-bottom:8px;margin-left:20px;margin-right:0px;margin-top:8px}#abs div.dateline{font-size:0.9em;font-style:italic;margin-bottom:6.5px;margin-left:20px;margin-right:0px;margin-top:6.5px}#abs blockquote.abstract{font-size:1.05em;margin-block-end:21.6px;margin-block-start:14.4px;margin-bottom:21.6px;margin-inline-end:40px;margin-inline-start:40px;background-color:white;border-left:0px;padding:0px}#abs div.metatable{font-size:0.95em!important;margin-bottom:19px;margin-left:20px;margin-right:0px;margin-top:0px;border:0px;padding:0px}#abs div.metatable tbody{vertical-align:middle}#abs tr{margin-top:0px;margin-bottom:0px}#abs td.tablecell{padding-top:0px;padding-bottom:0px;padding-right:6.5px;padding-left:0px;vertical-align:top;font-size:0.95em!important;margin-top:0px;margin-bottom:0px;border:0px}#abs td.tablecell.label{font-weight:400!important}#abs span.primary-subject{font-weight:700}#abs span.arxivid{font-weight:700}</style>
<div id=content-inner>
<div id=abs>
<div class=dateline>
[Submitted on 14 Aug 2024 (<a href=https://arxiv.org/abs/2408.08333v1>v1</a>), last revised 8 Jul 2025 (this version, v2)]</div>
<h1 class="title mathjax"><span class="descriptor sf-hidden">Title:</span>CodeMirage: Hallucinations in Code Generated by Large Language Models</h1>
<div class=authors><span class="descriptor sf-hidden">Authors:</span><a href="https://arxiv.org/search/cs?searchtype=author&amp;query=Agarwal,+V" rel=nofollow>Vibhor Agarwal</a>, <a href="https://arxiv.org/search/cs?searchtype=author&amp;query=Pei,+Y" rel=nofollow>Yulong Pei</a>, <a href="https://arxiv.org/search/cs?searchtype=author&amp;query=Alamir,+S" rel=nofollow>Salwa Alamir</a>, <a href="https://arxiv.org/search/cs?searchtype=author&amp;query=Liu,+X" rel=nofollow>Xiaomo Liu</a></div> <div id=download-button-info hidden class=sf-hidden>View a PDF of the paper titled CodeMirage: Hallucinations in Code Generated by Large Language Models, by Vibhor Agarwal and 3 other authors</div>
<a class="mobile-submission-download sf-hidden" href=https://arxiv.org/pdf/2408.08333>View PDF</a>
<a class="mobile-submission-download sf-hidden" href=https://arxiv.org/html/2408.08333v2>HTML (experimental)</a>
<blockquote class="abstract mathjax">
<span class="descriptor sf-hidden">Abstract:</span>Large Language Models (LLMs) have shown promising potentials in program generation and no-code automation. However, LLMs are prone to generate hallucinations, i.e., they generate text which sounds plausible but is incorrect. Although there has been a recent surge in research on LLM hallucinations for text generation, similar hallucination phenomenon can happen in code generation. Sometimes the generated code can have syntactical or logical errors as well as more advanced issues like security vulnerabilities, memory leaks, etc. Given the wide adaptation of LLMs to enhance efficiency in code generation and development in general, it becomes imperative to investigate hallucinations in code generation. To the best of our knowledge, this is the first attempt at studying hallucinations in the code generated by LLMs. We start by introducing the code hallucination definition and a comprehensive taxonomy of code hallucination types. We propose the first benchmark CodeMirage dataset for code hallucinations. The benchmark contains 1,137 GPT-3.5 generated hallucinated code snippets for Python programming problems from two base datasets - HumanEval and MBPP. We then propose the methodology for code hallucination detection and experiment with open source LLMs such as CodeLLaMA as well as OpenAI's GPT-3.5 and GPT-4 models using one-shot prompt. We find that GPT-4 performs the best on HumanEval dataset and gives comparable results to the fine-tuned CodeBERT baseline on MBPP dataset. Towards the end, we discuss various mitigation strategies for code hallucinations and conclude our work.
</blockquote>
<div class=metatable>
<table summary="Additional metadata"> <tbody><tr>
<td class="tablecell label">Comments:</td>
<td class="tablecell comments mathjax">Accepted at AutoMates @ IJCAI 2024</td>
</tr>
<tr>
<td class="tablecell label">Subjects:</td>
<td class="tablecell subjects">
<span class=primary-subject>Software Engineering (cs.SE)</span>; Artificial Intelligence (cs.AI); Computation and Language (cs.CL)</td>
<tr>
<td class="tablecell label">Cite as:</td>
<td class="tablecell arxivid"><span class=arxivid><a href=https://arxiv.org/abs/2408.08333>arXiv:2408.08333</a> [cs.SE]</span></td>
</tr>
<tr>
<td class="tablecell label">&nbsp;</td>
<td class="tablecell arxividv">(or <span class=arxivid>
<a href=https://arxiv.org/abs/2408.08333v2>arXiv:2408.08333v2</a> [cs.SE]</span> for this version)
</td>
</tr>
<tr>
<td class="tablecell label">&nbsp;</td>
<td class="tablecell arxivdoi"> <a href=https://doi.org/10.48550/arXiv.2408.08333 id=arxiv-doi-link>https://doi.org/10.48550/arXiv.2408.08333</a><div class=button-and-tooltip>
<button class=more-info aria-describedby=more-info-desc-1>
<svg height=15 role=presentation xmlns=http://www.w3.org/2000/svg viewBox="0 0 512 512"><path fill=currentColor d="M256 8C119.043 8 8 119.083 8 256c0 136.997 111.043 248 248 248s248-111.003 248-248C504 119.083 392.957 8 256 8zm0 110c23.196 0 42 18.804 42 42s-18.804 42-42 42-42-18.804-42-42 18.804-42 42-42zm56 254c0 6.627-5.373 12-12 12h-88c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h12v-64h-12c-6.627 0-12-5.373-12-12v-24c0-6.627 5.373-12 12-12h64c6.627 0 12 5.373 12 12v100h12c6.627 0 12 5.373 12 12v24z"></path></svg>
<span class=visually-hidden>Focus to learn more</span>
</button>
<div role=tooltip id=more-info-desc-1 class=sf-hidden>
arXiv-issued DOI via DataCite</div>
</div>
</td>
</table>
</div>
</div>
</div>
<div class=submission-history>
<h2>Submission history</h2> From: Vibhor Agarwal [<a href=https://arxiv.org/show-email/fa116746/2408.08333 rel=nofollow>view email</a>] <br> <strong><a href=https://arxiv.org/abs/2408.08333v1 rel=nofollow>[v1]</a></strong>
Wed, 14 Aug 2024 22:53:07 UTC (85 KB)<br>
<strong>[v2]</strong>
Tue, 8 Jul 2025 23:14:43 UTC (28 KB)<br>
</div>
</div>
<div class=extra-services> <div class=full-text>
<a name=other></a>
<span class="descriptor sf-hidden">Full-text links:</span>
<h2>Access Paper:</h2>
<ul>
<div id=download-button-info hidden class=sf-hidden>
View a PDF of the paper titled CodeMirage: Hallucinations in Code Generated by Large Language Models, by Vibhor Agarwal and 3 other authors</div><li><a href=https://arxiv.org/pdf/2408.08333 aria-describedby=download-button-info accesskey=f class="abs-button download-pdf">View PDF</a><li><a href=https://arxiv.org/html/2408.08333v2 class=abs-button id=latexml-download-link>HTML (experimental)</a><li><a href=https://arxiv.org/src/2408.08333 class="abs-button download-eprint">TeX Source
</a></ul>
<div class=abs-license><a href=http://creativecommons.org/licenses/by-nc-sa/4.0/ title="Rights to this article" class=has_license>
<img alt="license icon" role=presentation src=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAFAAAAAPCAIAAAD8q9/YAAABqklEQVRIx+WWMU8CMRTH3wewxRvZoGp0IpJOhDhwdMP1JhLHY3DSwU4kxAHYMIGlX+Gc1MXA4AQhpPEb8BX6FWq05Dyhd3Ju573ccGnea+/3/u+9K0AOTefGvoEfn4Lo07nuAMBwOLSGMcYwxhMxNs6r91XhsJAJXe3AEzHGGMfRhsynZ6fGf/o27fV6GQauX9TL5XJybUgpAaB91TYhS7lsXbayCowwMvJKKQkhAMAYU0oxxgCAECKlNCJXaTWs6tHDyDoXtg/7+W5djEZZnRNOSfC3A3fvuwAwm8201oQQz/O01vzLHMdRSgkhgiAwiwijsKrni7l1962XuA/azY41WXE5SucfBb69uwEAKaVSKjq3PM+jlEarmnOOMTZRL6/Pi9UiIfdWGeMg41Kzu+euvAkip1CYMeb7vlGYcy6E2CiMUiickHtrGf+q2FbsHxX+7GG06eEgCBzHAQDf98MeppSu12utdbPZrJxX9uzh/YH37+G4MZEauFavlUqlHE3piRgjdDAYDBKAXdc9Pjn6J//hcHT1+30rbcNtFIvFbN+08nWXzpV9AD9ivwErgn5oAAAAAElFTkSuQmCC>
<span>view license</span>
</a></div>
</div>
<div class=browse>
Current browse context: <div class=current>cs.SE</div>
<div class=prevnext>
<span class=arrow>
<a class="abs-button prev-url" href="https://arxiv.org/prevnext?id=2408.08333&amp;function=prev&amp;context=cs.SE" accesskey=p title="previous in cs.SE (accesskey p)" rel=nofollow>&lt;&nbsp;prev</a>
</span>
<span class=is-hidden-mobile>&nbsp; | &nbsp;</span> <span class=arrow>
<a class="abs-button next-url" href="https://arxiv.org/prevnext?id=2408.08333&amp;function=next&amp;context=cs.SE" accesskey=n title="next in cs.SE (accesskey n)" rel=nofollow>next&nbsp;&gt;</a>
</span><br>
</div><div class=list>
<a class="abs-button abs-button-grey abs-button-small context-new" href=https://arxiv.org/list/cs.SE/new rel=nofollow>new</a>
<span class=is-hidden-mobile> | </span>
<a class="abs-button abs-button-grey abs-button-small context-recent" href=https://arxiv.org/list/cs.SE/recent rel=nofollow>recent</a>
<span class=is-hidden-mobile> | </span><a class="abs-button abs-button-grey abs-button-small context-id" href=https://arxiv.org/list/cs.SE/2024-08 rel=nofollow>2024-08</a>
</div><div class=abs-switch-cat>
Change to browse by:
<div class="switch context-change">
<a href="https://arxiv.org/abs/2408.08333?context=cs" rel=nofollow>cs</a><br class=is-hidden-mobile>
<a class=subclass href="https://arxiv.org/abs/2408.08333?context=cs.AI" rel=nofollow>cs.AI</a><br class=is-hidden-mobile>
<a class=subclass href="https://arxiv.org/abs/2408.08333?context=cs.CL" rel=nofollow>cs.CL</a><br class=is-hidden-mobile>
</div>
</div>
</div>
<div class=extra-ref-cite>
<h3>References &amp; Citations</h3>
<ul>
<li><a class="abs-button abs-button-small cite-ads" href=https://ui.adsabs.harvard.edu/abs/arXiv:2408.08333>NASA ADS</a><li><a class="abs-button abs-button-small cite-google-scholar" href="https://scholar.google.com/scholar_lookup?arxiv_id=2408.08333" target=_blank rel=noopener>Google Scholar</a></li>
<li><a class="abs-button abs-button-small cite-semantic-scholar" href=https://api.semanticscholar.org/arXiv:2408.08333 target=_blank rel=noopener>Semantic Scholar</a></li>
</ul>
<div style=clear:both></div>
</div>
<div class=extra-ref-cite>
<span id=bib-cite-trigger class="bib-cite-button abs-button">export BibTeX citation</span>
<span id=bib-cite-loading hidden class=sf-hidden>Loading...</span>
</div>
<div id=bib-cite-modal class="bib-modal sf-hidden" hidden>
</div><div class=bookmarks>
<div><h3>Bookmark</h3></div><a class="abs-button abs-button-grey abs-button-small" href="http://www.bibsonomy.org/BibtexHandler?requTask=upload&amp;url=https://arxiv.org/abs/2408.08333&amp;description=CodeMirage:%20Hallucinations%20in%20Code%20Generated%20by%20Large%20Language%20Models" title="Bookmark on BibSonomy">
<img src=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAABTVBMVEXn5ubf3t6/H2OioaKgn6DLysrHxsa4tre3traysLGxsLCqqKmmpKWioKGenJ2cmpuamJmYlpeWlJWVlJSUkpOQjo+Ib3qMiov+/v78/Pz6+vr5+Pn4+Pj29vb09PSrqal1cnTu7u7i4uLe3t7V1NWJh4fJyMnHxsfDwsO/vr+9vL3y8fG3treDXW3e3d2oqKilpKWkoqSjoqPY19fS0dGXlpfMy8uUkpTGxcXEw8ONjI28u7u4t7ezsbKxr7CjH1etq6yrqaqnpaampaWko6OjoaKioaGhn6Cgn5+fnZ6em52dm5ybmZqVk5SIWW2Rj5CPjY6Oi42Ni4z////9/f37+/t/fX75+floSlfv7+/p6eno5+jl5eXk4+Tj4+Pd3d3b29vZ2dnX19fT09PS0dLR0dGEa3XOzc7Nzc3KycrIx8i+vb53dHW6ubrr6up9Unu9AAAA0klEQVQYlWMIRgJRgeJsDMFSxnEwAYcsRXOGYBcLp5TgYMlsu2RPW2ZVBoZgfS1hs+BgS83oMLlgZbtghuDQhCSRmOBYEXet4OBU1xAGkF5tAePgBNvgYBU+LgewQLCMm5oee7CsEDcrL0QgONQgTZeHL9E4QwYqEJxrzyQmAWJABELVfSP8vIJhAiEqgmo6WSGO6WABE3kOb+ssIwXZ4FABRpBAjjSLXrCoJGdicLCsfx5QwNmQPzw408NABigbGaQRD3QpknetlEwZgpGBj00AAMhwRAoMDs/uAAAAAElFTkSuQmCC alt="BibSonomy logo">
</a>
<a class="abs-button abs-button-grey abs-button-small" href="https://reddit.com/submit?url=https://arxiv.org/abs/2408.08333&amp;title=CodeMirage:%20Hallucinations%20in%20Code%20Generated%20by%20Large%20Language%20Models" title="Bookmark on Reddit">
<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAMAAABhEH5lAAAAclBMVEWtra3/IQClpaX/hFJ7e3tzc3Nra2tjY2P/Yyn/597/zr05OTkxMTHe3t4pKSnW1tbOzs7Gxsb/vaX/GACcnJyUlJSMjIyEhIT/rYxaWlpSUlL///9KSkr39/dCQkL/jFrv7+//hFrn5+f/jGO9vb21tbWaFPpZAAAAxUlEQVQYlU2Q7VrDMAhGEcMyO6thNsH0ta5m9f5vUVxbJz/ycQLkPBD2aMY91TCD/lCRQU5zoDuapAGSo97Rq/gys2JF05KtjpbT7ZFakHAQSq3pIGIxT2SD1H6vXj65MAXzJufrE9B9dWjRMi0vnnN8eAce3y4An3pCxJ517QAt/qMk/Is8ONLaMGVt7Zvdzi/uNVSFmgv04DGtqilW1pQ0f4RfnZt9HNmKZX4u2BCPee0dDsuGjMs6DKOyou3kcwhqvv0AYpYbpE15FCsAAAAASUVORK5CYII=" alt="Reddit logo">
</a>
</div> </div>
<div id=labstabs>
<div class=labstabs><input type=radio name=tabs id=tabone checked value=on class=sf-hidden>
<label for=tabone>Bibliographic Tools</label>
<div class="tab labs-display-bib">
<h1>Bibliographic and Citation Tools</h1>
<div class=toggle>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class=switch>
<input id=bibex-toggle type=checkbox class="lab-toggle sf-hidden" data-script-url=/static/browse/0.3.4/bibex/bibex.js?20241202 value=on>
<span class=slider></span>
<span class=is-sr-only>Bibliographic Explorer Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id=label-for-bibex>Bibliographic Explorer</span> <em>(<a href=https://info.arxiv.org/labs/showcase.html#arxiv-bibliographic-explorer>What is the Explorer?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class=switch>
<input id=connectedpapers-toggle type=checkbox class="lab-toggle sf-hidden" data-script-url=/static/browse/0.3.4/js/connectedpapers.js aria-labelledby=label-for-connected-papers value=on>
<span class=slider></span>
<span class=is-sr-only>Connected Papers Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id=label-for-connected-papers>Connected Papers</span> <em>(<a href=https://www.connectedpapers.com/about target=_blank>What is Connected Papers?</a>)</em>
</div>
</div><div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class=switch>
<input id=litmaps-toggle type=checkbox class="lab-toggle sf-hidden" data-script-url=/static/browse/0.3.4/js/litmaps.js?20210617 aria-labelledby=label-for-litmaps value=on>
<span class=slider></span>
<span class=is-sr-only>Litmaps Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id=label-for-litmaps>Litmaps</span> <em>(<a href=https://www.litmaps.co/ target=_blank>What is Litmaps?</a>)</em>
</div>
</div>
<div class="columns is-mobile lab-row">
<div class="column lab-switch">
<label class=switch>
<input id=scite-toggle type=checkbox class="lab-toggle sf-hidden" data-script-url=/static/browse/0.3.4/js/scite.js?20210617 aria-labelledby=label-for-scite value=on>
<span class=slider></span>
<span class=is-sr-only>scite.ai Toggle</span>
</label>
</div>
<div class="column lab-name">
<span id=label-for-scite>scite Smart Citations</span> <em>(<a href=https://www.scite.ai/ target=_blank>What are Smart Citations?</a>)</em>
</div>
</div>
</div>
<div class="labs-content-placeholder labs-display" style=display:none></div>
<div style=min-height:15px id=connectedpapers-output></div>
<div style=min-height:15px id=litmaps-open-in></div>
<div style=min-height:15px id=scite-open-in></div>
</div>
<input type=radio name=tabs id=tabtwo value=on class=sf-hidden>
<label for=tabtwo>Code, Data, Media</label>
<div class="tab sf-hidden">
</div>
<input type=radio name=tabs id=labstabs-demos-input value=on class=sf-hidden>
<label for=labstabs-demos-input id=labstabs-demos-label>Demos</label>
<div class="tab sf-hidden">
</div>
<input type=radio name=tabs id=tabfour value=on class=sf-hidden>
<label for=tabfour>Related Papers</label>
<div class="tab sf-hidden">
</div>
<input type=radio name=tabs id=tabfive value=on class=sf-hidden>
<label for=tabfive>
About arXivLabs
</label>
<div class="tab sf-hidden">
</div>
</div>
</div>
<div class=endorsers>
<a href=https://arxiv.org/auth/show-endorsers/2408.08333 class=endorser-who rel=nofollow>Which authors of this paper are endorsers?</a> |
<a id=mathjax_toggle href=javascript:void(0)>Disable MathJax</a> (<a href=https://info.arxiv.org/help/mathjax.html>What is MathJax?</a>)
<span class=help style=font-style:normal;float:right;margin-top:0;margin-right:1em></span>
</div>
</div>
</div>
</main>
<footer style=clear:both>
<div class="columns is-desktop" role=navigation aria-label=Secondary style="margin:-0.75em -0.75em 0.75em -0.75em">
<div class=column style=padding:0>
<div class=columns>
<div class=column>
<ul style=list-style:none;line-height:2>
<li><a href=https://info.arxiv.org/about>About</a></li>
<li><a href=https://info.arxiv.org/help>Help</a></li>
</ul>
</div>
<div class=column>
<ul style=list-style:none;line-height:2>
<li>
<svg xmlns=http://www.w3.org/2000/svg viewBox="0 0 512 512" class="icon filter-black" role=presentation><title>contact arXiv</title><desc>Click here to contact arXiv</desc><path d="M502.3 190.8c3.9-3.1 9.7-.2 9.7 4.7V400c0 26.5-21.5 48-48 48H48c-26.5 0-48-21.5-48-48V195.6c0-5 5.7-7.8 9.7-4.7 22.4 17.4 52.1 39.5 154.1 113.6 21.1 15.4 56.7 47.8 92.2 47.6 35.7.3 72-32.8 92.3-47.6 102-74.1 131.6-96.3 154-113.7zM256 320c23.2.4 56.6-29.2 73.4-41.4 132.7-96.3 142.8-104.7 173.4-128.7 5.8-4.5 9.2-11.5 9.2-18.9v-19c0-26.5-21.5-48-48-48H48C21.5 64 0 85.5 0 112v19c0 7.4 3.4 14.3 9.2 18.9 30.6 23.9 40.7 32.4 173.4 128.7 16.8 12.2 50.2 41.8 73.4 41.4z"></path></svg>
<a href=https://info.arxiv.org/help/contact.html> Contact</a>
</li>
<li>
<svg xmlns=http://www.w3.org/2000/svg viewBox="0 0 512 512" class="icon filter-black" role=presentation><title>subscribe to arXiv mailings</title><desc>Click here to subscribe</desc><path d="M476 3.2L12.5 270.6c-18.1 10.4-15.8 35.6 2.2 43.2L121 358.4l287.3-253.2c5.5-4.9 13.3 2.6 8.6 8.3L176 407v80.5c0 23.6 28.5 32.9 42.5 15.8L282 426l124.6 52.2c14.2 6 30.4-2.9 33-18.2l72-432C515 7.8 493.3-6.8 476 3.2z"></path></svg>
<a href=https://info.arxiv.org/help/subscribe> Subscribe</a>
</li>
</ul>
</div>
</div>
</div>
<div class=column style=padding:0>
<div class=columns>
<div class=column>
<ul style=list-style:none;line-height:2>
<li><a href=https://info.arxiv.org/help/license/index.html>Copyright</a></li>
<li><a href=https://info.arxiv.org/help/policies/privacy_policy.html>Privacy Policy</a></li>
</ul>
</div>
<div class="column sorry-app-links">
<ul style=list-style:none;line-height:2>
<li><a href=https://info.arxiv.org/help/web_accessibility.html>Web Accessibility Assistance</a></li>
<li>
<p class=help>
<a class=a11y-main-link href=https://status.arxiv.org/ target=_blank>arXiv Operational Status <svg xmlns=http://www.w3.org/2000/svg viewBox="0 0 256 512" class="icon filter-dark_grey" role=presentation><path d="M224.3 273l-136 136c-9.4 9.4-24.6 9.4-33.9 0l-22.6-22.6c-9.4-9.4-9.4-24.6 0-33.9l96.4-96.4-96.4-96.4c-9.4-9.4-9.4-24.6 0-33.9L54.3 103c9.4-9.4 24.6-9.4 33.9 0l136 136c9.5 9.4 9.5 24.6.1 34z"></path></svg></a><br>
</p>
</li>
</ul>
</div>
</div>
</div>
</div>
</footer>
</div>
<a href=# style=visibility:hidden id=__TIMER_INJECTION_FLAG__{a8cf72f7-09b7-4cd4-9aaa-7a023bf09916} class=sf-hidden></a>
Reference in New Issue View Git Blame Copy Permalink