From 19c5da86d126c14a0c5c55096bb51ca120ebce63 Mon Sep 17 00:00:00 2001 From: Madhav Varshney Date: Sat, 23 May 2020 01:18:29 -0700 Subject: [PATCH] Fix duplicates in scraper due to extra whitespace --- scrape_term.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrape_term.py b/scrape_term.py index 3d40193..f41cadf 100644 --- a/scrape_term.py +++ b/scrape_term.py @@ -62,6 +62,7 @@ def parse(content, db): tables = soup.find_all('table', {'class': 'TblCourses'}) for t in tables: + # TODO: verify whether replacing spaces yields correct dept names in all scenarios dept = t['dept'].replace(' ', '') dept_desc = t['dept-desc'] @@ -73,7 +74,7 @@ def parse(content, db): if cols: for i, c in enumerate(cols): a = c.find('a') - cols[i] = a.get_text() if a else cols[i].get_text() + cols[i] = (a.get_text() if a else cols[i].get_text()).strip() try: key = get_key(f'{cols[0] if cols[0] else cols[1]}')[0]