') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); [microNPU] Set output tolerance of codegen and network tests to 0 by lhutton1 · Pull Request #10675 · apache/tvm · GitHub
Skip to content

[microNPU] Set output tolerance of codegen and network tests to 0 - #10675

Merged
Mousius merged 2 commits into
apache:mainfrom
lhutton1:reference-kernel
Apr 7, 2022
Merged

[microNPU] Set output tolerance of codegen and network tests to 0#10675
Mousius merged 2 commits into
apache:mainfrom
lhutton1:reference-kernel

Conversation

@lhutton1

@lhutton1lhutton1 commented Mar 18, 2022

Copy link
Copy Markdown
Contributor

After the recent upgrade of Tensorflow to 2.6, we are now able to use reference kernels in order to verify the output. Thus, removing the tolerances previously added.

Additionally, the network tests have been altered to use TFLite as a reference, rather than TVM.

cc @manupa-arm@ekalda@NicolaLancellotti@dchauhan-arm

@github-actions
github-actionsBot requested a review from manupakMarch 18, 2022 16:15
@lhutton1
lhutton1force-pushed the reference-kernel branch 2 times, most recently from cd65869 to 4797434CompareMarch 24, 2022 10:16
@NicolaLancellotti

NicolaLancellotti commented Mar 24, 2022

Copy link
Copy Markdown
Contributor

Can you check if we need these two output tolerances?

tvm/tests/python/contrib/test_ethosu/test_codegen.py

Line 508 in 1c40a45

output_tolerance=1ifoperator_type=="MAX"else0,

tvm/tests/python/contrib/test_ethosu/test_codegen.py

Line 797 in 1c40a45

ethosu_mod, input_data, output_data, accel_type, output_tolerance=1

@lhutton1

lhutton1 commented Mar 24, 2022

Copy link
Copy Markdown
ContributorAuthor

Thanks for pointing these out @NicolaLancellotti! The identity tolerance cannot be changed since the output can be slightly different compared to the numpy reference due to quantization (cc @ekalda). However, we should be able to remove the tolerance from max. I'll update after #10762 to save an additional rebase :)

@lhutton1
[microNPU] Set output tolerance of codegen and network tests to 0
563e6f0
After the recent upgrade of Tensorflow to 2.6, we are now able to use
reference kernels in order to verify the output. Thus, removing the
tolerances previously added.
Additionally, the network tests have been altered to use TFLite as a
reference, rather than TVM.
Change-Id: Idcf408b84c67456d16da5d61f4865eb8b6042554
@lhutton1

lhutton1 commented Apr 5, 2022

Copy link
Copy Markdown
ContributorAuthor

After #10889 this should now pass CI

NicolaLancellotti
NicolaLancellotti approved these changes Apr 6, 2022
View reviewed changes

@NicolaLancellottiNicolaLancellotti left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LGTM!

@lhutton1
move utility function to infra to fix i386
24f501a
Change-Id: I174cc48a47c75590f76e0ee2a9634547b2442e87
Mousius
Mousius approved these changes Apr 7, 2022
View reviewed changes

@MousiusMousius left a comment

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LGTM!

@Mousius
Mousius merged commit 05bb482 into apache:mainApr 7, 2022
@Mousius

Mousius commented Apr 7, 2022

Copy link
Copy Markdown
Member

Nice to see zero tolerance @lhutton1 / @NicolaLancellotti 😸

pfk-beta pushed a commit to pfk-beta/tvm that referenced this pull request Apr 11, 2022
@lhutton1pfk-beta
[microNPU] Set output tolerance of codegen and network tests to 0 (ap…
f3ec892
…ache#10675)
After the recent upgrade of Tensorflow to 2.6, we are now able to use
reference kernels in order to verify the output. Thus, removing the
tolerances previously added.
Additionally, the network tests have been altered to use TFLite as a
reference, rather than TVM.
mehrdadh pushed a commit to mehrdadh/tvm that referenced this pull request Apr 11, 2022
@lhutton1@mehrdadh
[microNPU] Set output tolerance of codegen and network tests to 0 (ap…
22178c4
…ache#10675)
After the recent upgrade of Tensorflow to 2.6, we are now able to use
reference kernels in order to verify the output. Thus, removing the
tolerances previously added.
Additionally, the network tests have been altered to use TFLite as a
reference, rather than TVM.
altanh pushed a commit to altanh/tvm that referenced this pull request Apr 28, 2022
@lhutton1@altanh
[microNPU] Set output tolerance of codegen and network tests to 0 (ap…
c6d8ade
…ache#10675)
After the recent upgrade of Tensorflow to 2.6, we are now able to use
reference kernels in order to verify the output. Thus, removing the
tolerances previously added.
Additionally, the network tests have been altered to use TFLite as a
reference, rather than TVM.
@lhutton1
lhutton1 deleted the reference-kernel branch May 9, 2022 21:43
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@MousiusMousiusMousius approved these changes
@NicolaLancellottiNicolaLancellottiNicolaLancellotti approved these changes
@manupakmanupakAwaiting requested review from manupak

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

3 participants

@lhutton1@NicolaLancellotti@Mousius