[DISCARDED] Add Wikipedia as a new data source (wikipedia_fetch.py)

README.md

@@ -57,6 +57,7 @@ Quantifying/
 │ ├── 1-fetch/
 │ ├── 2-process/
 │ ├── 3-report/
+ └── plot.py # Data visualisations with matplotlib
 │ └── shared.py
 ├── .cc-metadata.yml
 ├── .flake8 # Python tool configuration

scripts/1-fetch/wikipedia_fetch.py

@@ -0,0 +1,87 @@
+import requests
+from typing import Dict
+
+WIKI_API = "https://en.wikipedia.org/w/api.php"
+
+# Add a User-Agent to avoid 403 errors
+HEADERS = {
+ "User-Agent": "QuantifyingCommonsBot/1.0 (https://github.com/YOUR_USERNAME/quantifying)"
+}
+
+
+def get_site_statistics() -> Dict[str, int]:
+ """
+ Fetch general statistics from Wikipedia.
+
+ Returns:
+ dict: Dictionary containing:
+ - articles: number of articles
+ - pages: number of pages
+ - edits: number of edits
+ - users: number of users
+ - images: number of images
+ """
+ params = {
+ "action": "query",
+ "meta": "siteinfo",
+ "siprop": "statistics|rightsinfo",
+ "format": "json"
+ }
+
+ response = requests.get(WIKI_API, headers=HEADERS, params=params)
+ response.raise_for_status()
+ data = response.json()
+
+ stats = data.get("query", {}).get("statistics", {})
+
+ return {
+ "articles": stats.get("articles", 0),
+ "pages": stats.get("pages", 0),
+ "edits": stats.get("edits", 0),
+ "users": stats.get("users", 0),
+ "images": stats.get("images", 0)
+ }
+
+
+def search_articles_by_license(license_keyword: str, limit: int = 10) -> Dict[str, int]:
+ """
+ Search Wikipedia articles containing a specific Creative Commons license keyword.
+
+ Args:
+ license_keyword (str): e.g., "CC BY-SA 4.0"
+ limit (int): Number of results to fetch
+
+ Returns:
+ dict: Dictionary with count and sample articles
+ """
+ params = {
+ "action": "query",
+ "list": "search",
+ "srsearch": license_keyword,
+ "srlimit": limit,
+ "format": "json"
+ }
+
+ response = requests.get(WIKI_API, headers=HEADERS, params=params)
+ response.raise_for_status()
+ data = response.json()
+
+ search_results = data.get("query", {}).get("search", [])
+ return {
+ "count": len(search_results),
+ "sample_titles": [item["title"] for item in search_results]
+ }
+
+
+if __name__ == "__main__":
+ print("Wikipedia Site Statistics:")
+ stats = get_site_statistics()
+ print(stats)
+
+ license_query = "Creative Commons"
+ print(f"\nArticles mentioning '{license_query}':")
+ results = search_articles_by_license(license_query, limit=5)
+ print(results)
+
+
+

scripts/shared.py

@@ -5,7 +5,7 @@
 
 # Third-party
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
-from pandas import PeriodIndex
+from pandas import
 
 
 class QuantifyingException(Exception):

sources.md

@@ -174,3 +174,35 @@ adjust video parameters, and obtain search results.
  - API key required
  - Query limit: depends on the type and number of requests
  - Data available through JSON format
+
+## 📖 Wikipedia Data Source
+
+Quantifying now supports fetching data from Wikipedia as an additional source alongside GitHub and Google Custom Search.
+
+### Available Statistics
+
+- **Number of articles** – Total articles on Wikipedia.
+- **Number of pages** – Total pages, including non-article pages.
+- **Number of edits** – Total edits across Wikipedia.
+- **Number of users** – Total registered users.
+- **Number of images** – Total uploaded images.
+- **Keyword-based counts** – Number of articles referencing specific Creative Commons licenses or keywords.
+
+### Example Usage
+
+```python
+from scripts.wikipedia_fetch import get_site_statistics, search_articles_count, fetch_cc_related_statistics
+
+# General Wikipedia statistics
+stats = get_site_statistics()
+print("Wikipedia Site Stats:", stats)
+
+# Count articles containing a specific keyword
+cc_articles = search_articles_count("Creative Commons")
+print("Articles with 'Creative Commons':", cc_articles)
+
+# Fetch counts for various Creative Commons licenses
+cc_stats = fetch_cc_related_statistics()
+for license_name, count in cc_stats.items():
+ print(f"{license_name}: {count}")
+