Complete MiniProject2

pgoedec1.ipynb → .ipynb_checkpoints/nmansou4-checkpoint.ipynb

@@ -15,27 +15,22 @@
  "import time\n",
  "import datetime\n",
  "import requests\n",
- "from bs4 import BeautifulSoup\n",
  "\n",
  "dbname = \"fdac18mp2\" #please use this database\n",
- "collname = \"tgoedecke\" #please modify so you store data in your collection\n",
- "my_char = 'c'\n",
- "\n",
+ "collname = \"glprj_nmansou4\" #please modify so you store data in your collection\n",
  "# beginning page index\n",
- "begin = \"1\"\n",
+ "begin = \"0\"\n",
  "client = pymongo.MongoClient()\n",
  "\n",
  "db = client[dbname]\n",
  "coll = db[collname]\n",
  "\n",
  "\n",
- "gitlab_url = \"https://gitlab.com/api/v4/projects?archived=false&membership=false&order_by=created_at&owned=false&page=\" + begin + \\\n",
+ "beginurl = \"https://gitlab.com/api/v4/projects?archived=false&membership=false&order_by=created_at&owned=false&page=\" + begin + \\\n",
  " \"&per_page=99&simple=false&sort=desc&starred=false&statistics=false&with_custom_attributes=false&with_issues_enabled=false&with_merge_requests_enabled=false\"\n",
  "\n",
- "gleft = 20\n",
  "\n",
- "source_url = \"https://sourceforge.net/directory/?q=\" + my_char + \"&sort=name&page=\"\n",
- "rest_url = \"https://sourceforge.net/rest/p/\"\n",
+ "gleft = 0\n",
  "\n",
  "header = {'per_page': 99}\n",
  "\n",
@@ -49,47 +44,15 @@
  " time .sleep(60)\n",
  " return left\n",
  "\n",
- "def project_exists(url):\n",
- " r = requests.get(url)\n",
- " if r.status_code == 200:\n",
- " return True\n",
- " return False\n",
- "\n",
- "def get_source(url, coll, rest):\n",
- " page = 1\n",
- " project_count = 0\n",
- " while True:\n",
- " resp = requests.get(url + str(page))\n",
- " text = resp.text\n",
- " soup = BeautifulSoup(text, 'html.parser')\n",
- " if re.search('No results found.', soup.get_text()):\n",
- " return\n",
- "\n",
- " for link in soup.find_all(class_=\"project-icon\", href=True):\n",
- " name = re.findall('/projects/([A-Za-z0-9\\-]*)', link.get('href'))\n",
- " name = name[0] if name else None\n",
- " if name is not None and name.lower().startswith(my_char):\n",
- " resp = requests.get(rest + name)\n",
- " if resp.status_code == 200:\n",
- " info = json.loads(resp.text)\n",
- " info['forge'] = 'sourceforge'\n",
- " coll.insert_one(info)\n",
- " project_count += 1\n",
- " if project_count >= 50:\n",
- " return\n",
- " page += 1\n",
- " return\n",
- "\n",
  "# send queries and extract urls \n",
- "def get_gitlab(url, coll):\n",
+ "def get(url, coll):\n",
  "\n",
  " global gleft\n",
  " global header\n",
  " global bginnum\n",
  " gleft = wait(gleft)\n",
  " values = []\n",
  " size = 0\n",
- " project_count = 0\n",
  "\n",
  " try:\n",
  " r = requests .get(url, headers=header)\n",
@@ -105,14 +68,8 @@
  " array = json.loads(t)\n",
  " \n",
  " for el in array:\n",
- " if el['name'].lower().startswith(my_char):\n",
- " if project_exists(el['http_url_to_repo']):\n",
- " project_count += 1\n",
- " el['forge'] = 'gitlab'\n",
- " coll.insert_one(el)\n",
- " if project_count >= 50:\n",
- " return\n",
- " \n",
+ " coll.insert(el)\n",
+ " \n",
  " #next page\n",
  " while ('; rel=\"next\"' in lll):\n",
  " gleft = int(r.headers.get('RateLimit-Remaining'))\n",
@@ -131,19 +88,13 @@
  " t = r.text\n",
  " array1 = json.loads(t)\n",
  " for el in array1:\n",
- " if el['name'].lower().startswith(my_char):\n",
- " if project_exists(el['http_url_to_repo']):\n",
- " project_count += 1\n",
- " el['forge'] = 'gitlab'\n",
- " coll.insert_one(el)\n",
- " if project_count >= 50:\n",
- " return\n",
+ " coll.insert(el)\n",
  " else:\n",
  " sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
  " return \n",
  " except requests.exceptions.ConnectionError:\n",
  " sys.stderr.write('could not get ' + url + '\\n')\n",
- "\n",
+ "\n",
  " else:\n",
  " sys.stderr.write(\"url can not found:\\n\" + url + '\\n')\n",
  " return\n",
@@ -154,11 +105,7 @@
  " sys.stderr.write(url + ';' + str(e) + '\\n')\n",
  " \n",
  "#start retrieving \n",
- "get_gitlab(gitlab_url,coll)\n",
- "get_source(source_url, coll, rest_url)\n",
- "#print collected data\n",
- "for doc in coll.find({}):\n",
- " print(doc)"
+ "get(beginurl,coll)"
  ]
  },
  {