') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Fix test_cublas and relax tolerance by areusch · Pull Request #8108 · apache/tvm · GitHub
Skip to content

Fix test_cublas and relax tolerance - #8108

Closed
areusch wants to merge 1 commit into
apache:mainfrom
areusch:fix-ci-gpu-tests
Closed

Fix test_cublas and relax tolerance#8108
areusch wants to merge 1 commit into
apache:mainfrom
areusch:fix-ci-gpu-tests

Conversation

@areusch

@areuschareusch commented May 21, 2021

Copy link
Copy Markdown
Contributor
  • Allows test to pass on ci-gpu with new 18.04 image.

@tkonolige@tqchen

Andrew Reusch
Fix test_cublas and relax tolerance.
1b89b37
 * Allows test to pass on ci-gpu with new 18.04 image.
@comaniac

comaniac commented May 21, 2021

Copy link
Copy Markdown
Contributor

FYI: It's possible that this error is caused by the CUBLAS math flag:

tvm/src/runtime/contrib/cublas/cublas.cc

Line 38 in 8131364

inlinevoidCUBLASTryEnableTensorCore(cublasHandle_t hdl) {

According to the CUBLAS document, this flag is being deprecated. We also made some tests previously around this flag, and found that it is effective even for float32, meaning that CUBLAS kernel internally casts float32 to float16, does the computation, and casts the results back. As a result, this flag may introduce accuracy issue.

@areusch

areusch commented May 24, 2021

Copy link
Copy Markdown
ContributorAuthor

@comaniac i was sending this through CI for @tkonolige as he was out friday. I'll let him reply to your comment.

@tkonolige

tkonolige commented May 24, 2021

Copy link
Copy Markdown
Contributor

@comaniac Disabling the flag makes the tests pass. What should we do here? Accept lower accuracy for performance?

@comaniac

comaniac commented May 24, 2021

Copy link
Copy Markdown
Contributor

@comaniac Disabling the flag makes the tests pass. What should we do here? Accept lower accuracy for performance?

I personally prefer to keep the accuracy, because it seems not right to tolerate 1e-2 for a single batch_matmul op. It means the end-to-end accuracy of all models with cublas.batch_matmul may be larger than 1e-2. cc @Hzfengsy@Laurawly as they added this flag at the time it hasn't been deprecated.

@Hzfengsy

Hzfengsy commented May 25, 2021

Copy link
Copy Markdown
Member

I also prefer to keep accuracy. Just as @comaniac said, 1e-2 is too low for larger end-to-end workloads

@tkonoligetkonolige mentioned this pull request May 25, 2021
Merged
@areuschareusch mentioned this pull request May 25, 2021
Closed
17 tasks
@areusch

areusch commented May 27, 2021

Copy link
Copy Markdown
ContributorAuthor

superseded by #8130

@areuschareusch closed this May 27, 2021
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

No reviews

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

4 participants

@areusch@comaniac@tkonolige@Hzfengsy