Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
CaedenPH wants to merge 6 commits into from
Closed

Fix fetch anime and play and mark as unbroken #8972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
190afd4
updating DIRECTORY.md
Aug 7, 2023
c1d7692
Merge branch 'master' of https://github.com/caedenph/python
CaedenPH Aug 16, 2023
8bfdd14
fix(get-amazon-product-data): Remove whitespace in headers
CaedenPH Aug 16, 2023
d8d4402
refactor(get-amazon-product-data): Don't print to_csv
CaedenPH Aug 16, 2023
7943625
fix(fetch-anime-and-play): Unmark as BROKEN
CaedenPH Aug 16, 2023
bc84328
updating DIRECTORY.md
Aug 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions DIRECTORY.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
* [Daily Horoscope](web_programming/daily_horoscope.py)
* [Download Images From Google Query](web_programming/download_images_from_google_query.py)
* [Emails From Url](web_programming/emails_from_url.py)
* [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
* [Fetch Bbc News](web_programming/fetch_bbc_news.py)
* [Fetch Github Info](web_programming/fetch_github_info.py)
* [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down
73 changes: 38 additions & 35 deletions ...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
Take an url and
return list of anime after scraping the site.

>>> type(search_scraper("demon_slayer"))
>>> type(search_scraper("naruto"))
<class 'list'>

Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

# get list of anime
anime_ul = soup.find("ul", {"class": "items"})
if anime_ul is None or isinstance(anime_ul, NavigableString):
msg = f"Could not find and anime with name {anime_name}"
raise ValueError(msg)
anime_li = anime_ul.children

# for each anime, insert to list. the name and url.
anime_list = []
for anime in anime_li:
if not isinstance(anime, NavigableString):
try:
anime_url, anime_title = (
anime.find("a")["href"],
anime.find("a")["title"],
)
anime_list.append(
{
"title": anime_title,
"url": anime_url,
}
)
except (NotFoundErr, KeyError):
pass
if isinstance(anime, Tag):
anime_url = anime.find("a")
if anime_url is None or isinstance(anime_url, NavigableString):
continue
anime_title = anime.find("a")
if anime_title is None or isinstance(anime_title, NavigableString):
continue

anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

# With this id. get the episode list.
episode_page_ul = soup.find("ul", {"id": "episode_related"})
if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
msg = f"Could not find any anime eposiodes with name {anime_name}"
raise ValueError(msg)
episode_page_li = episode_page_ul.children

episode_list = []
for episode in episode_page_li:
try:
if not isinstance(episode, NavigableString):
episode_list.append(
{
"title": episode.find("div", {"class": "name"}).text.replace(
" ", ""
),
"url": episode.find("a")["href"],
}
)
except (KeyError, NotFoundErr):
pass
if isinstance(episode, Tag):
url = episode.find("a")
if url is None or isinstance(url, NavigableString):
continue
title = episode.find("div", {"class": "name"})
if title is None or isinstance(title, NavigableString):
continue

episode_list.append(
{"title": title.text.replace(" ", ""), "url": url["href"]}
)

return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

soup = BeautifulSoup(response.text, "html.parser")

try:
episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
except (KeyError, NotFoundErr) as e:
raise e
url = soup.find("iframe", {"id": "playerframe"})
if url is None or isinstance(url, NavigableString):
msg = f"Could not find url and download url from {episode_endpoint}"
raise RuntimeError(msg)

episode_url = url["src"]
if not isinstance(episode_url, str):
msg = f"Could not find url and download url from {episode_endpoint}"
raise RuntimeError(msg)
download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down
12 changes: 7 additions & 5 deletions web_programming/get_amazon_product_data.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
"""
url = f"https://www.amazon.in/laptop/s?k={product}"
header = {
"User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
"(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
),
"Accept-Language": "en-US, en;q=0.5",
}
soup = BeautifulSoup(requests.get(url, headers=header).text)
soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
# Initialize a Pandas dataframe with the column titles
data_frame = DataFrame(
columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
except ValueError:
discount = float("nan")
except AttributeError:
pass
data_frame.loc[len(data_frame.index)] = [
continue
data_frame.loc[str(len(data_frame.index))] = [
product_title,
product_link,
product_price,
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

CaedenPH
wants to merge
6
commits into

from

 Closed


Fix fetch anime and play and mark as unbroken
#8972
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
190afd4

updating DIRECTORY.md

Aug 7, 2023
c1d7692

Merge branch 'master' of https://github.com/caedenph/python


CaedenPH Aug 16, 2023
8bfdd14

fix(get-amazon-product-data): Remove whitespace in headers


CaedenPH Aug 16, 2023
d8d4402

refactor(get-amazon-product-data): Don't print to_csv


CaedenPH Aug 16, 2023
7943625

fix(fetch-anime-and-play): Unmark as BROKEN


CaedenPH Aug 16, 2023
bc84328

updating DIRECTORY.md

Aug 16, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
DIRECTORY.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
 * [Daily Horoscope](web_programming/daily_horoscope.py)
 * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
 * [Emails From Url](web_programming/emails_from_url.py)
 * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
 * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
 * [Fetch Github Info](web_programming/fetch_github_info.py)
 * [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down

73 changes: 38 additions & 35 deletions
...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
 Take an url and
 return list of anime after scraping the site.

 >>> type(search_scraper("demon_slayer"))
 >>> type(search_scraper("naruto"))
 <class 'list'>

 Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

 # get list of anime
 anime_ul = soup.find("ul", {"class": "items"})
 if anime_ul is None or isinstance(anime_ul, NavigableString):
 msg = f"Could not find and anime with name {anime_name}"
 raise ValueError(msg)
 anime_li = anime_ul.children

 # for each anime, insert to list. the name and url.
 anime_list = []
 for anime in anime_li:
 if not isinstance(anime, NavigableString):
 try:
 anime_url, anime_title = (
 anime.find("a")["href"],
 anime.find("a")["title"],
 )
 anime_list.append(
 {
 "title": anime_title,
 "url": anime_url,
 }
 )
 except (NotFoundErr, KeyError):
 pass
 if isinstance(anime, Tag):
 anime_url = anime.find("a")
 if anime_url is None or isinstance(anime_url, NavigableString):
 continue
 anime_title = anime.find("a")
 if anime_title is None or isinstance(anime_title, NavigableString):
 continue

 anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

 return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

 # With this id. get the episode list.
 episode_page_ul = soup.find("ul", {"id": "episode_related"})
 if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
 msg = f"Could not find any anime eposiodes with name {anime_name}"
 raise ValueError(msg)
 episode_page_li = episode_page_ul.children

 episode_list = []
 for episode in episode_page_li:
 try:
 if not isinstance(episode, NavigableString):
 episode_list.append(
 {
 "title": episode.find("div", {"class": "name"}).text.replace(
 " ", ""
 ),
 "url": episode.find("a")["href"],
 }
 )
 except (KeyError, NotFoundErr):
 pass
 if isinstance(episode, Tag):
 url = episode.find("a")
 if url is None or isinstance(url, NavigableString):
 continue
 title = episode.find("div", {"class": "name"})
 if title is None or isinstance(title, NavigableString):
 continue

 episode_list.append(
 {"title": title.text.replace(" ", ""), "url": url["href"]}
 )

 return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

 soup = BeautifulSoup(response.text, "html.parser")

 try:
 episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 except (KeyError, NotFoundErr) as e:
 raise e
 url = soup.find("iframe", {"id": "playerframe"})
 if url is None or isinstance(url, NavigableString):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)

 episode_url = url["src"]
 if not isinstance(episode_url, str):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

 return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down

12 changes: 7 additions & 5 deletions
web_programming/get_amazon_product_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 """
 url = f"https://www.amazon.in/laptop/s?k={product}"
 header = {
 "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
 (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
 "User-Agent": (
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
 "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
 ),
 "Accept-Language": "en-US, en;q=0.5",
 }
 soup = BeautifulSoup(requests.get(url, headers=header).text)
 soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
 # Initialize a Pandas dataframe with the column titles
 data_frame = DataFrame(
 columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 except ValueError:
 discount = float("nan")
 except AttributeError:
pass
 data_frame.loc[len(data_frame.index)] = [
continue
 data_frame.loc[str(len(data_frame.index))] = [
 product_title,
 product_link,
 product_price,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

CaedenPH
wants to merge
6
commits into

from

 Closed


Fix fetch anime and play and mark as unbroken
#8972
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
190afd4

updating DIRECTORY.md

Aug 7, 2023
c1d7692

Merge branch 'master' of https://github.com/caedenph/python


CaedenPH Aug 16, 2023
8bfdd14

fix(get-amazon-product-data): Remove whitespace in headers


CaedenPH Aug 16, 2023
d8d4402

refactor(get-amazon-product-data): Don't print to_csv


CaedenPH Aug 16, 2023
7943625

fix(fetch-anime-and-play): Unmark as BROKEN


CaedenPH Aug 16, 2023
bc84328

updating DIRECTORY.md

Aug 16, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
DIRECTORY.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
 * [Daily Horoscope](web_programming/daily_horoscope.py)
 * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
 * [Emails From Url](web_programming/emails_from_url.py)
 * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
 * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
 * [Fetch Github Info](web_programming/fetch_github_info.py)
 * [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down

73 changes: 38 additions & 35 deletions
...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
 Take an url and
 return list of anime after scraping the site.

 >>> type(search_scraper("demon_slayer"))
 >>> type(search_scraper("naruto"))
 <class 'list'>

 Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

 # get list of anime
 anime_ul = soup.find("ul", {"class": "items"})
 if anime_ul is None or isinstance(anime_ul, NavigableString):
 msg = f"Could not find and anime with name {anime_name}"
 raise ValueError(msg)
 anime_li = anime_ul.children

 # for each anime, insert to list. the name and url.
 anime_list = []
 for anime in anime_li:
 if not isinstance(anime, NavigableString):
 try:
 anime_url, anime_title = (
 anime.find("a")["href"],
 anime.find("a")["title"],
 )
 anime_list.append(
 {
 "title": anime_title,
 "url": anime_url,
 }
 )
 except (NotFoundErr, KeyError):
 pass
 if isinstance(anime, Tag):
 anime_url = anime.find("a")
 if anime_url is None or isinstance(anime_url, NavigableString):
 continue
 anime_title = anime.find("a")
 if anime_title is None or isinstance(anime_title, NavigableString):
 continue

 anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

 return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

 # With this id. get the episode list.
 episode_page_ul = soup.find("ul", {"id": "episode_related"})
 if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
 msg = f"Could not find any anime eposiodes with name {anime_name}"
 raise ValueError(msg)
 episode_page_li = episode_page_ul.children

 episode_list = []
 for episode in episode_page_li:
 try:
 if not isinstance(episode, NavigableString):
 episode_list.append(
 {
 "title": episode.find("div", {"class": "name"}).text.replace(
 " ", ""
 ),
 "url": episode.find("a")["href"],
 }
 )
 except (KeyError, NotFoundErr):
 pass
 if isinstance(episode, Tag):
 url = episode.find("a")
 if url is None or isinstance(url, NavigableString):
 continue
 title = episode.find("div", {"class": "name"})
 if title is None or isinstance(title, NavigableString):
 continue

 episode_list.append(
 {"title": title.text.replace(" ", ""), "url": url["href"]}
 )

 return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

 soup = BeautifulSoup(response.text, "html.parser")

 try:
 episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 except (KeyError, NotFoundErr) as e:
 raise e
 url = soup.find("iframe", {"id": "playerframe"})
 if url is None or isinstance(url, NavigableString):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)

 episode_url = url["src"]
 if not isinstance(episode_url, str):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

 return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down

12 changes: 7 additions & 5 deletions
web_programming/get_amazon_product_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 """
 url = f"https://www.amazon.in/laptop/s?k={product}"
 header = {
 "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
 (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
 "User-Agent": (
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
 "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
 ),
 "Accept-Language": "en-US, en;q=0.5",
 }
 soup = BeautifulSoup(requests.get(url, headers=header).text)
 soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
 # Initialize a Pandas dataframe with the column titles
 data_frame = DataFrame(
 columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 except ValueError:
 discount = float("nan")
 except AttributeError:
pass
 data_frame.loc[len(data_frame.index)] = [
continue
 data_frame.loc[str(len(data_frame.index))] = [
 product_title,
 product_link,
 product_price,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

CaedenPH
wants to merge
6
commits into

from

 Closed


Fix fetch anime and play and mark as unbroken
#8972
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
190afd4

updating DIRECTORY.md

Aug 7, 2023
c1d7692

Merge branch 'master' of https://github.com/caedenph/python


CaedenPH Aug 16, 2023
8bfdd14

fix(get-amazon-product-data): Remove whitespace in headers


CaedenPH Aug 16, 2023
d8d4402

refactor(get-amazon-product-data): Don't print to_csv


CaedenPH Aug 16, 2023
7943625

fix(fetch-anime-and-play): Unmark as BROKEN


CaedenPH Aug 16, 2023
bc84328

updating DIRECTORY.md

Aug 16, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
DIRECTORY.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
 * [Daily Horoscope](web_programming/daily_horoscope.py)
 * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
 * [Emails From Url](web_programming/emails_from_url.py)
 * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
 * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
 * [Fetch Github Info](web_programming/fetch_github_info.py)
 * [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down

73 changes: 38 additions & 35 deletions
...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
 Take an url and
 return list of anime after scraping the site.

 >>> type(search_scraper("demon_slayer"))
 >>> type(search_scraper("naruto"))
 <class 'list'>

 Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

 # get list of anime
 anime_ul = soup.find("ul", {"class": "items"})
 if anime_ul is None or isinstance(anime_ul, NavigableString):
 msg = f"Could not find and anime with name {anime_name}"
 raise ValueError(msg)
 anime_li = anime_ul.children

 # for each anime, insert to list. the name and url.
 anime_list = []
 for anime in anime_li:
 if not isinstance(anime, NavigableString):
 try:
 anime_url, anime_title = (
 anime.find("a")["href"],
 anime.find("a")["title"],
 )
 anime_list.append(
 {
 "title": anime_title,
 "url": anime_url,
 }
 )
 except (NotFoundErr, KeyError):
 pass
 if isinstance(anime, Tag):
 anime_url = anime.find("a")
 if anime_url is None or isinstance(anime_url, NavigableString):
 continue
 anime_title = anime.find("a")
 if anime_title is None or isinstance(anime_title, NavigableString):
 continue

 anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

 return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

 # With this id. get the episode list.
 episode_page_ul = soup.find("ul", {"id": "episode_related"})
 if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
 msg = f"Could not find any anime eposiodes with name {anime_name}"
 raise ValueError(msg)
 episode_page_li = episode_page_ul.children

 episode_list = []
 for episode in episode_page_li:
 try:
 if not isinstance(episode, NavigableString):
 episode_list.append(
 {
 "title": episode.find("div", {"class": "name"}).text.replace(
 " ", ""
 ),
 "url": episode.find("a")["href"],
 }
 )
 except (KeyError, NotFoundErr):
 pass
 if isinstance(episode, Tag):
 url = episode.find("a")
 if url is None or isinstance(url, NavigableString):
 continue
 title = episode.find("div", {"class": "name"})
 if title is None or isinstance(title, NavigableString):
 continue

 episode_list.append(
 {"title": title.text.replace(" ", ""), "url": url["href"]}
 )

 return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

 soup = BeautifulSoup(response.text, "html.parser")

 try:
 episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 except (KeyError, NotFoundErr) as e:
 raise e
 url = soup.find("iframe", {"id": "playerframe"})
 if url is None or isinstance(url, NavigableString):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)

 episode_url = url["src"]
 if not isinstance(episode_url, str):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

 return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down

12 changes: 7 additions & 5 deletions
web_programming/get_amazon_product_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 """
 url = f"https://www.amazon.in/laptop/s?k={product}"
 header = {
 "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
 (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
 "User-Agent": (
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
 "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
 ),
 "Accept-Language": "en-US, en;q=0.5",
 }
 soup = BeautifulSoup(requests.get(url, headers=header).text)
 soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
 # Initialize a Pandas dataframe with the column titles
 data_frame = DataFrame(
 columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 except ValueError:
 discount = float("nan")
 except AttributeError:
pass
 data_frame.loc[len(data_frame.index)] = [
continue
 data_frame.loc[str(len(data_frame.index))] = [
 product_title,
 product_link,
 product_price,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

CaedenPH
wants to merge
6
commits into

from

 Closed


Fix fetch anime and play and mark as unbroken
#8972
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
190afd4

updating DIRECTORY.md

Aug 7, 2023
c1d7692

Merge branch 'master' of https://github.com/caedenph/python


CaedenPH Aug 16, 2023
8bfdd14

fix(get-amazon-product-data): Remove whitespace in headers


CaedenPH Aug 16, 2023
d8d4402

refactor(get-amazon-product-data): Don't print to_csv


CaedenPH Aug 16, 2023
7943625

fix(fetch-anime-and-play): Unmark as BROKEN


CaedenPH Aug 16, 2023
bc84328

updating DIRECTORY.md

Aug 16, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
DIRECTORY.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
 * [Daily Horoscope](web_programming/daily_horoscope.py)
 * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
 * [Emails From Url](web_programming/emails_from_url.py)
 * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
 * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
 * [Fetch Github Info](web_programming/fetch_github_info.py)
 * [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down

73 changes: 38 additions & 35 deletions
...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
 Take an url and
 return list of anime after scraping the site.

 >>> type(search_scraper("demon_slayer"))
 >>> type(search_scraper("naruto"))
 <class 'list'>

 Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

 # get list of anime
 anime_ul = soup.find("ul", {"class": "items"})
 if anime_ul is None or isinstance(anime_ul, NavigableString):
 msg = f"Could not find and anime with name {anime_name}"
 raise ValueError(msg)
 anime_li = anime_ul.children

 # for each anime, insert to list. the name and url.
 anime_list = []
 for anime in anime_li:
 if not isinstance(anime, NavigableString):
 try:
 anime_url, anime_title = (
 anime.find("a")["href"],
 anime.find("a")["title"],
 )
 anime_list.append(
 {
 "title": anime_title,
 "url": anime_url,
 }
 )
 except (NotFoundErr, KeyError):
 pass
 if isinstance(anime, Tag):
 anime_url = anime.find("a")
 if anime_url is None or isinstance(anime_url, NavigableString):
 continue
 anime_title = anime.find("a")
 if anime_title is None or isinstance(anime_title, NavigableString):
 continue

 anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

 return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

 # With this id. get the episode list.
 episode_page_ul = soup.find("ul", {"id": "episode_related"})
 if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
 msg = f"Could not find any anime eposiodes with name {anime_name}"
 raise ValueError(msg)
 episode_page_li = episode_page_ul.children

 episode_list = []
 for episode in episode_page_li:
 try:
 if not isinstance(episode, NavigableString):
 episode_list.append(
 {
 "title": episode.find("div", {"class": "name"}).text.replace(
 " ", ""
 ),
 "url": episode.find("a")["href"],
 }
 )
 except (KeyError, NotFoundErr):
 pass
 if isinstance(episode, Tag):
 url = episode.find("a")
 if url is None or isinstance(url, NavigableString):
 continue
 title = episode.find("div", {"class": "name"})
 if title is None or isinstance(title, NavigableString):
 continue

 episode_list.append(
 {"title": title.text.replace(" ", ""), "url": url["href"]}
 )

 return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

 soup = BeautifulSoup(response.text, "html.parser")

 try:
 episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 except (KeyError, NotFoundErr) as e:
 raise e
 url = soup.find("iframe", {"id": "playerframe"})
 if url is None or isinstance(url, NavigableString):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)

 episode_url = url["src"]
 if not isinstance(episode_url, str):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

 return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down

12 changes: 7 additions & 5 deletions
web_programming/get_amazon_product_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 """
 url = f"https://www.amazon.in/laptop/s?k={product}"
 header = {
 "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
 (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
 "User-Agent": (
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
 "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
 ),
 "Accept-Language": "en-US, en;q=0.5",
 }
 soup = BeautifulSoup(requests.get(url, headers=header).text)
 soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
 # Initialize a Pandas dataframe with the column titles
 data_frame = DataFrame(
 columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 except ValueError:
 discount = float("nan")
 except AttributeError:
pass
 data_frame.loc[len(data_frame.index)] = [
continue
 data_frame.loc[str(len(data_frame.index))] = [
 product_title,
 product_link,
 product_price,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

CaedenPH
wants to merge
6
commits into

from

 Closed


Fix fetch anime and play and mark as unbroken
#8972
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
190afd4

updating DIRECTORY.md

Aug 7, 2023
c1d7692

Merge branch 'master' of https://github.com/caedenph/python


CaedenPH Aug 16, 2023
8bfdd14

fix(get-amazon-product-data): Remove whitespace in headers


CaedenPH Aug 16, 2023
d8d4402

refactor(get-amazon-product-data): Don't print to_csv


CaedenPH Aug 16, 2023
7943625

fix(fetch-anime-and-play): Unmark as BROKEN


CaedenPH Aug 16, 2023
bc84328

updating DIRECTORY.md

Aug 16, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
DIRECTORY.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
 * [Daily Horoscope](web_programming/daily_horoscope.py)
 * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
 * [Emails From Url](web_programming/emails_from_url.py)
 * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
 * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
 * [Fetch Github Info](web_programming/fetch_github_info.py)
 * [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down

73 changes: 38 additions & 35 deletions
...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
 Take an url and
 return list of anime after scraping the site.

 >>> type(search_scraper("demon_slayer"))
 >>> type(search_scraper("naruto"))
 <class 'list'>

 Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

 # get list of anime
 anime_ul = soup.find("ul", {"class": "items"})
 if anime_ul is None or isinstance(anime_ul, NavigableString):
 msg = f"Could not find and anime with name {anime_name}"
 raise ValueError(msg)
 anime_li = anime_ul.children

 # for each anime, insert to list. the name and url.
 anime_list = []
 for anime in anime_li:
 if not isinstance(anime, NavigableString):
 try:
 anime_url, anime_title = (
 anime.find("a")["href"],
 anime.find("a")["title"],
 )
 anime_list.append(
 {
 "title": anime_title,
 "url": anime_url,
 }
 )
 except (NotFoundErr, KeyError):
 pass
 if isinstance(anime, Tag):
 anime_url = anime.find("a")
 if anime_url is None or isinstance(anime_url, NavigableString):
 continue
 anime_title = anime.find("a")
 if anime_title is None or isinstance(anime_title, NavigableString):
 continue

 anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

 return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

 # With this id. get the episode list.
 episode_page_ul = soup.find("ul", {"id": "episode_related"})
 if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
 msg = f"Could not find any anime eposiodes with name {anime_name}"
 raise ValueError(msg)
 episode_page_li = episode_page_ul.children

 episode_list = []
 for episode in episode_page_li:
 try:
 if not isinstance(episode, NavigableString):
 episode_list.append(
 {
 "title": episode.find("div", {"class": "name"}).text.replace(
 " ", ""
 ),
 "url": episode.find("a")["href"],
 }
 )
 except (KeyError, NotFoundErr):
 pass
 if isinstance(episode, Tag):
 url = episode.find("a")
 if url is None or isinstance(url, NavigableString):
 continue
 title = episode.find("div", {"class": "name"})
 if title is None or isinstance(title, NavigableString):
 continue

 episode_list.append(
 {"title": title.text.replace(" ", ""), "url": url["href"]}
 )

 return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

 soup = BeautifulSoup(response.text, "html.parser")

 try:
 episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 except (KeyError, NotFoundErr) as e:
 raise e
 url = soup.find("iframe", {"id": "playerframe"})
 if url is None or isinstance(url, NavigableString):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)

 episode_url = url["src"]
 if not isinstance(episode_url, str):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

 return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down

12 changes: 7 additions & 5 deletions
web_programming/get_amazon_product_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 """
 url = f"https://www.amazon.in/laptop/s?k={product}"
 header = {
 "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
 (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
 "User-Agent": (
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
 "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
 ),
 "Accept-Language": "en-US, en;q=0.5",
 }
 soup = BeautifulSoup(requests.get(url, headers=header).text)
 soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
 # Initialize a Pandas dataframe with the column titles
 data_frame = DataFrame(
 columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 except ValueError:
 discount = float("nan")
 except AttributeError:
pass
 data_frame.loc[len(data_frame.index)] = [
continue
 data_frame.loc[str(len(data_frame.index))] = [
 product_title,
 product_link,
 product_price,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

CaedenPH
wants to merge
6
commits into

from

 Closed


Fix fetch anime and play and mark as unbroken
#8972
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
190afd4

updating DIRECTORY.md

Aug 7, 2023
c1d7692

Merge branch 'master' of https://github.com/caedenph/python


CaedenPH Aug 16, 2023
8bfdd14

fix(get-amazon-product-data): Remove whitespace in headers


CaedenPH Aug 16, 2023
d8d4402

refactor(get-amazon-product-data): Don't print to_csv


CaedenPH Aug 16, 2023
7943625

fix(fetch-anime-and-play): Unmark as BROKEN


CaedenPH Aug 16, 2023
bc84328

updating DIRECTORY.md

Aug 16, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
DIRECTORY.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
 * [Daily Horoscope](web_programming/daily_horoscope.py)
 * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
 * [Emails From Url](web_programming/emails_from_url.py)
 * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
 * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
 * [Fetch Github Info](web_programming/fetch_github_info.py)
 * [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down

73 changes: 38 additions & 35 deletions
...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
 Take an url and
 return list of anime after scraping the site.

 >>> type(search_scraper("demon_slayer"))
 >>> type(search_scraper("naruto"))
 <class 'list'>

 Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

 # get list of anime
 anime_ul = soup.find("ul", {"class": "items"})
 if anime_ul is None or isinstance(anime_ul, NavigableString):
 msg = f"Could not find and anime with name {anime_name}"
 raise ValueError(msg)
 anime_li = anime_ul.children

 # for each anime, insert to list. the name and url.
 anime_list = []
 for anime in anime_li:
 if not isinstance(anime, NavigableString):
 try:
 anime_url, anime_title = (
 anime.find("a")["href"],
 anime.find("a")["title"],
 )
 anime_list.append(
 {
 "title": anime_title,
 "url": anime_url,
 }
 )
 except (NotFoundErr, KeyError):
 pass
 if isinstance(anime, Tag):
 anime_url = anime.find("a")
 if anime_url is None or isinstance(anime_url, NavigableString):
 continue
 anime_title = anime.find("a")
 if anime_title is None or isinstance(anime_title, NavigableString):
 continue

 anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

 return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

 # With this id. get the episode list.
 episode_page_ul = soup.find("ul", {"id": "episode_related"})
 if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
 msg = f"Could not find any anime eposiodes with name {anime_name}"
 raise ValueError(msg)
 episode_page_li = episode_page_ul.children

 episode_list = []
 for episode in episode_page_li:
 try:
 if not isinstance(episode, NavigableString):
 episode_list.append(
 {
 "title": episode.find("div", {"class": "name"}).text.replace(
 " ", ""
 ),
 "url": episode.find("a")["href"],
 }
 )
 except (KeyError, NotFoundErr):
 pass
 if isinstance(episode, Tag):
 url = episode.find("a")
 if url is None or isinstance(url, NavigableString):
 continue
 title = episode.find("div", {"class": "name"})
 if title is None or isinstance(title, NavigableString):
 continue

 episode_list.append(
 {"title": title.text.replace(" ", ""), "url": url["href"]}
 )

 return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

 soup = BeautifulSoup(response.text, "html.parser")

 try:
 episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 except (KeyError, NotFoundErr) as e:
 raise e
 url = soup.find("iframe", {"id": "playerframe"})
 if url is None or isinstance(url, NavigableString):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)

 episode_url = url["src"]
 if not isinstance(episode_url, str):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

 return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down

12 changes: 7 additions & 5 deletions
web_programming/get_amazon_product_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 """
 url = f"https://www.amazon.in/laptop/s?k={product}"
 header = {
 "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
 (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
 "User-Agent": (
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
 "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
 ),
 "Accept-Language": "en-US, en;q=0.5",
 }
 soup = BeautifulSoup(requests.get(url, headers=header).text)
 soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
 # Initialize a Pandas dataframe with the column titles
 data_frame = DataFrame(
 columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 except ValueError:
 discount = float("nan")
 except AttributeError:
pass
 data_frame.loc[len(data_frame.index)] = [
continue
 data_frame.loc[str(len(data_frame.index))] = [
 product_title,
 product_link,
 product_price,
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Fix fetch anime and play and mark as unbroken#8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

CaedenPH
wants to merge
6
commits into

from

 Closed


Fix fetch anime and play and mark as unbroken
#8972
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
190afd4

updating DIRECTORY.md

Aug 7, 2023
c1d7692

Merge branch 'master' of https://github.com/caedenph/python


CaedenPH Aug 16, 2023
8bfdd14

fix(get-amazon-product-data): Remove whitespace in headers


CaedenPH Aug 16, 2023
d8d4402

refactor(get-amazon-product-data): Don't print to_csv


CaedenPH Aug 16, 2023
7943625

fix(fetch-anime-and-play): Unmark as BROKEN


CaedenPH Aug 16, 2023
bc84328

updating DIRECTORY.md

Aug 16, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
DIRECTORY.md

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1212,6 +1212,7 @@
 * [Daily Horoscope](web_programming/daily_horoscope.py)
 * [Download Images From Google Query](web_programming/download_images_from_google_query.py)
 * [Emails From Url](web_programming/emails_from_url.py)
 * [Fetch Anime And Play](web_programming/fetch_anime_and_play.py)
 * [Fetch Bbc News](web_programming/fetch_bbc_news.py)
 * [Fetch Github Info](web_programming/fetch_github_info.py)
 * [Fetch Jobs](web_programming/fetch_jobs.py)
Expand Down

73 changes: 38 additions & 35 deletions
...rogramming/fetch_anime_and_play.py.BROKEN → web_programming/fetch_anime_and_play.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
from xml.dom import NotFoundErr

import requests
from bs4 import BeautifulSoup, NavigableString
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
Expand All@@ -13,7 +11,7 @@ def search_scraper(anime_name: str) -> list:
 Take an url and
 return list of anime after scraping the site.

 >>> type(search_scraper("demon_slayer"))
 >>> type(search_scraper("naruto"))
 <class 'list'>

 Args:
Expand DownExpand Up@@ -41,25 +39,23 @@ def search_scraper(anime_name: str) -> list:

 # get list of anime
 anime_ul = soup.find("ul", {"class": "items"})
 if anime_ul is None or isinstance(anime_ul, NavigableString):
 msg = f"Could not find and anime with name {anime_name}"
 raise ValueError(msg)
 anime_li = anime_ul.children

 # for each anime, insert to list. the name and url.
 anime_list = []
 for anime in anime_li:
 if not isinstance(anime, NavigableString):
 try:
 anime_url, anime_title = (
 anime.find("a")["href"],
 anime.find("a")["title"],
 )
 anime_list.append(
 {
 "title": anime_title,
 "url": anime_url,
 }
 )
 except (NotFoundErr, KeyError):
 pass
 if isinstance(anime, Tag):
 anime_url = anime.find("a")
 if anime_url is None or isinstance(anime_url, NavigableString):
 continue
 anime_title = anime.find("a")
 if anime_title is None or isinstance(anime_title, NavigableString):
 continue

 anime_list.append({"title": anime_title["title"], "url": anime_url["href"]})

 return anime_list

Expand DownExpand Up@@ -93,22 +89,24 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

 # With this id. get the episode list.
 episode_page_ul = soup.find("ul", {"id": "episode_related"})
 if episode_page_ul is None or isinstance(episode_page_ul, NavigableString):
 msg = f"Could not find any anime eposiodes with name {anime_name}"
 raise ValueError(msg)
 episode_page_li = episode_page_ul.children

 episode_list = []
 for episode in episode_page_li:
 try:
 if not isinstance(episode, NavigableString):
 episode_list.append(
 {
 "title": episode.find("div", {"class": "name"}).text.replace(
 " ", ""
 ),
 "url": episode.find("a")["href"],
 }
 )
 except (KeyError, NotFoundErr):
 pass
 if isinstance(episode, Tag):
 url = episode.find("a")
 if url is None or isinstance(url, NavigableString):
 continue
 title = episode.find("div", {"class": "name"})
 if title is None or isinstance(title, NavigableString):
 continue

 episode_list.append(
 {"title": title.text.replace(" ", ""), "url": url["href"]}
 )

 return episode_list

Expand DownExpand Up@@ -140,11 +138,16 @@ def get_anime_episode(episode_endpoint: str) -> list:

 soup = BeautifulSoup(response.text, "html.parser")

 try:
 episode_url = soup.find("iframe", {"id": "playerframe"})["src"]
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"
 except (KeyError, NotFoundErr) as e:
 raise e
 url = soup.find("iframe", {"id": "playerframe"})
 if url is None or isinstance(url, NavigableString):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)

 episode_url = url["src"]
 if not isinstance(episode_url, str):
 msg = f"Could not find url and download url from {episode_endpoint}"
 raise RuntimeError(msg)
 download_url = episode_url.replace("/embed/", "/playlist/") + ".m3u8"

 return [f"{BASE_URL}{episode_url}", f"{BASE_URL}{download_url}"]

Expand Down

12 changes: 7 additions & 5 deletions
web_programming/get_amazon_product_data.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,11 +19,13 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 """
 url = f"https://www.amazon.in/laptop/s?k={product}"
 header = {
 "User-Agent": """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36
 (KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36""",
 "User-Agent": (
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
 "(KHTML, like Gecko)Chrome/44.0.2403.157 Safari/537.36"
 ),
 "Accept-Language": "en-US, en;q=0.5",
 }
 soup = BeautifulSoup(requests.get(url, headers=header).text)
 soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml")
 # Initialize a Pandas dataframe with the column titles
 data_frame = DataFrame(
 columns=[
Expand DownExpand Up@@ -74,8 +76,8 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
 except ValueError:
 discount = float("nan")
 except AttributeError:
pass
 data_frame.loc[len(data_frame.index)] = [
continue
 data_frame.loc[str(len(data_frame.index))] = [
 product_title,
 product_link,
 product_price,
Expand Down