- Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathoricon.py
More file actions
Latest commit
208 lines (179 loc) · 7.07 KB
/
Copy pathoricon.py
File metadata and controls
208 lines (179 loc) · 7.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
importre
importsys
frompathlibimportPath
fromshutilimportwhich
fromurllib.parseimportunquote, urljoin
importrequests
frombs4importBeautifulSoup
defget_webname(url):
returnunquote(url.split('?')[0].split('/')[-1])
defget(url):
r=requests.get(url)
r.encodings='shift-jis'
returnBeautifulSoup(r.content, 'html.parser')
defget_jpeg_quality(f):
fromsubprocessimportcheck_output
output=check_output(['magick', 'identify', '-format', '%Q;%[jpeg:sampling-factor]', f])
quality, sampling_factor=output.decode('utf8').split(';')
returnint(quality), sampling_factor
defbytes_to_kb(bytes):
bytes=int(bytes)
returnf'{bytes/1024:.3f} KB'
defdownload(url_or_res, f):
ifisinstance(url_or_res, str):
r=requests.get(url_or_res, stream=True)
else:
r=url_or_res
withf.open('wb') asfio:
forchunkinr.iter_content(chunk_size=8192):
ifchunk:
fio.write(chunk)
defget_orig(url, save_dir='.', test_mode=False, bad_file='delete'):
defcheck_quality(f):
size=f.stat().st_size
q, sampling_factor=get_jpeg_quality(f)
print(f'{bytes_to_kb(size)}, q{q}, {sampling_factor}')
ifq==85andsampling_factor=='2x2,1x1,1x1':
return'bad'
ifq>85:
return'good'
return'not sure'
save_dir=Path(save_dir)
print(f'Getting {url}')
web_name=get_webname(url)
f=save_dir/web_name
download(url, f)
print(' First try: ', end='')
old_quality=check_quality(f)
ifold_quality=='good':
f.rename(f.with_name(f'{f.stem}_orig.jpg'))
print(f' Find original. Stop.')
returnTrue
ifold_quality=='bad':
print(f' Likely re-compressed.')
else:
print(f' Not sure. Try to get a different one anyway.')
filesize=f.stat().st_size# cache filesize
tries=0
whileTrue:
tries+=1
r=requests.get(url, stream=True)
ifr.headers['Content-length'] ==str(filesize):
print(f' Remote is still the same size.')
else:
print(f' Got a different file: ', end='')
savef=f.with_name(f'{f.stem}_orig.jpg')
download(r, savef)
new_quality=check_quality(savef)
new_filesize=savef.stat().st_size
iftest_mode:
print(f' Test mode. So keep both files.')
f.rename(f.with_name(f'{f.stem}_{filesize}.jpg'))
savef.rename(f.with_name(f'{f.stem}_{new_filesize}.jpg'))
returnTrue
# potential results:
# bad -> good, not sure -> good: keep new
ifold_quality=='bad'andnew_quality=='good' \
orold_quality=='not sure'andnew_quality=='good':
print(f' New file is original. Stop and cleanup.')
ifbad_file=='delete':
f.unlink()
elifbad_file=='keep':
f.rename(f.with_name(f'{f.stem}_bad.jpg'))
elifbad_file=='move_to_subfolder':
(save_dir/'tobedel').mkdir(exist_ok=True)
f.rename(save_dir/'tobedel'/f.name)
savef.rename(f.with_name(f'{f.stem}_orig.jpg'))
returnTrue
# bad -> not sure, bad -> bad, not sure -> not sure, not sure -> bad: keep both
print(' Not sure which one is better. Save both. Please check yourself.')
f.rename(f.with_name(f'{f.stem}_{filesize}.jpg'))
savef.rename(f.with_name(f'{f.stem}_{new_filesize}.jpg'))
returnTrue
iftries>100:
print(f' Failed to get a different version of {url} after 100 tries.')
returnTrue
defget_image_from_photo_page(soup):
if (ele:=soup.find('meta', {'property': 'og:image'})) andele.has_attr('content'):
url=ele['content']
url=re.sub(r'cdn-cgi/image/[^/]+/upimg', 'upimg', url)
returnurl
else:
returnsoup.select_one('div#main_photo img')['src']
defsingle(url):
ifre.search(r'oricon\.co\.jp/news/\d+/photo/\d+', url):
img_url=get_image_from_photo_page(get(url))
elif'contents.oricon.co.jp'inurl:
img_url=url
else:
print(f'{url}: not a valid URL for single mode.')
returnFalse
get_orig(img_url)
defmain(url):
img_url_candidates= []
ifre.search(r'oricon\.co\.jp/news/', url):
print(f'{url}: news type')
url=re.sub(r'/news/(\d+)/*.+$', r'/news/\1/photo/1/', url)
print(f'Getting image from {url}')
soup=get(url)
img_url_candidates.append(get_image_from_photo_page(soup))
forainsoup.select(('div.photo_slider li > a')):
new_url=urljoin(url, a['href'])
ifnew_url==url:
continue
print(f'Getting image from {new_url}')
soup2=get(new_url)
img_url_candidates.append(get_image_from_photo_page(soup2))
elifm:=re.search(r'oricon\.co\.jp/(photo|special)/\d+', url):
page_type=m[1]
# https://www.oricon.co.jp/special/785/
url=re.sub(r'/(photo|special)/(\d+)/*.+$', r'/\1/\2/', url)
print(f'{url}: {page_type} type')
img_urls= {}
whileTrue:
print(f'Getting image from {url}')
soup=get(url)
forimginsoup.find_all('img'):
img_url=None
forattrin ['data-original', 'src']:
ifimg.has_attr(attr):
img_url=img[attr]
break
assertimg_urlisnotNone
if (m:=re.search(r'^(.+/(?:photo|special)/img/\d+/\d+/detail/img)(\d+)(/.+$)', img_url)):
ifnotm[3] inimg_urls:
img_urls[m[3]] =m[1]
else:
assertimg_urls[m[3]] ==m[1]
ifsoup.select_one('a.pager-next'):
url=urljoin(url, soup.select_one('a.pager-next')['href'])
else:
break
forsuffix, prefixinimg_urls.items():
forsizein [1500, 660, 480, 200, 100]:
img_url=f'{prefix}{size}{suffix}'
ifrequests.head(img_url).status_code==200:
print(f'Find a valid image: {img_url}')
img_url_candidates.append(img_url)
break
else:
print(f'{url}: not a valid URL.')
return
img_url_candidates=list(dict.fromkeys(img_url_candidates))
forphoto_urlinimg_url_candidates:
get_orig(photo_url)
if__name__=='__main__':
# detect magick
ifnotwhich('magick'):
print('magick is not installed. Please install it.')
sys.exit(1)
iflen(sys.argv) <2:
print('Usage: oricon.py <url>')
sys.exit(1)
else:
iflen(sys.argv) ==3andsys.argv[1] =='single':
url=sys.argv[2]
single(url)
else:
url=sys.argv[1]
main(url)