diff options
author | Rogério Brito <rbrito@ime.usp.br> | 2014-01-19 02:38:23 -0200 |
---|---|---|
committer | Rogério Brito <rbrito@ime.usp.br> | 2014-01-19 02:38:23 -0200 |
commit | e8cd8c4bd832446f1971215b9fedc4531555dc1a (patch) | |
tree | 51df61a563e242d643ae536fbb829f70880c947c /devscripts | |
parent | ca4d08063804fb264eb0ae9cc57894198f66e1fb (diff) | |
download | youtube-dl-e8cd8c4bd832446f1971215b9fedc4531555dc1a.zip youtube-dl-e8cd8c4bd832446f1971215b9fedc4531555dc1a.tar.gz youtube-dl-e8cd8c4bd832446f1971215b9fedc4531555dc1a.tar.bz2 |
Imported Upstream version 2014.01.17.2
Diffstat (limited to 'devscripts')
-rw-r--r-- | devscripts/bash-completion.in | 2 | ||||
-rw-r--r-- | devscripts/check-porn.py | 44 | ||||
-rwxr-xr-x | devscripts/gh-pages/update-feed.py | 92 | ||||
-rwxr-xr-x | devscripts/make_readme.py | 10 | ||||
-rwxr-xr-x | devscripts/release.sh | 2 |
5 files changed, 98 insertions, 52 deletions
diff --git a/devscripts/bash-completion.in b/devscripts/bash-completion.in index 3af87a3..28bd237 100644 --- a/devscripts/bash-completion.in +++ b/devscripts/bash-completion.in @@ -6,7 +6,7 @@ __youtube_dl() prev="${COMP_WORDS[COMP_CWORD-1]}" opts="{{flags}}" keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" - fileopts="-a|--batch-file|--download-archive|--cookies" + fileopts="-a|--batch-file|--download-archive|--cookies|--load-info" diropts="--cache-dir" if [[ ${prev} =~ ${fileopts} ]]; then diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 63401fe..86aa37b 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -3,6 +3,9 @@ """ This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check if we are not 'age_limit' tagging some porn site + +A second approach implemented relies on a list of porn domains, to activate it +pass the list filename as the only argument """ # Allow direct execution @@ -11,25 +14,42 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import get_testcases +from youtube_dl.utils import compat_urllib_parse_urlparse from youtube_dl.utils import compat_urllib_request +if len(sys.argv) > 1: + METHOD = 'LIST' + LIST = open(sys.argv[1]).read().decode('utf8').strip() +else: + METHOD = 'EURISTIC' + for test in get_testcases(): - try: - webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() - except: - print('\nFail: {0}'.format(test['name'])) - continue + if METHOD == 'EURISTIC': + try: + webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() + except: + print('\nFail: {0}'.format(test['name'])) + continue + + webpage = webpage.decode('utf8', 'replace') + + RESULT = 'porn' in webpage.lower() + + elif METHOD == 'LIST': + domain = compat_urllib_parse_urlparse(test['url']).netloc + if not domain: + print('\nFail: {0}'.format(test['name'])) + continue + domain = '.'.join(domain.split('.')[-2:]) - webpage = webpage.decode('utf8', 'replace') + RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST) - if 'porn' in webpage.lower() and ('info_dict' not in test - or 'age_limit' not in test['info_dict'] - or test['info_dict']['age_limit'] != 18): + if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] + or test['info_dict']['age_limit'] != 18): print('\nPotential missing age_limit check: {0}'.format(test['name'])) - elif 'porn' not in webpage.lower() and ('info_dict' in test and - 'age_limit' in test['info_dict'] and - test['info_dict']['age_limit'] == 18): + elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] + and test['info_dict']['age_limit'] == 18): print('\nPotential false negative: {0}'.format(test['name'])) else: diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py index 16571a9..0ba15ae 100755 --- a/devscripts/gh-pages/update-feed.py +++ b/devscripts/gh-pages/update-feed.py @@ -1,56 +1,76 @@ #!/usr/bin/env python3 import datetime - +import io +import json import textwrap -import json -atom_template=textwrap.dedent("""\ - <?xml version='1.0' encoding='utf-8'?> - <atom:feed xmlns:atom="http://www.w3.org/2005/Atom"> - <atom:title>youtube-dl releases</atom:title> - <atom:id>youtube-dl-updates-feed</atom:id> - <atom:updated>@TIMESTAMP@</atom:updated> - @ENTRIES@ - </atom:feed>""") - -entry_template=textwrap.dedent(""" - <atom:entry> - <atom:id>youtube-dl-@VERSION@</atom:id> - <atom:title>New version @VERSION@</atom:title> - <atom:link href="http://rg3.github.io/youtube-dl" /> - <atom:content type="xhtml"> - <div xmlns="http://www.w3.org/1999/xhtml"> - Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a> - </div> - </atom:content> - <atom:author> - <atom:name>The youtube-dl maintainers</atom:name> - </atom:author> - <atom:updated>@TIMESTAMP@</atom:updated> - </atom:entry> - """) +atom_template = textwrap.dedent("""\ + <?xml version="1.0" encoding="utf-8"?> + <feed xmlns="http://www.w3.org/2005/Atom"> + <link rel="self" href="http://rg3.github.io/youtube-dl/update/releases.atom" /> + <title>youtube-dl releases</title> + <id>https://yt-dl.org/feed/youtube-dl-updates-feed</id> + <updated>@TIMESTAMP@</updated> + @ENTRIES@ + </feed>""") -now = datetime.datetime.now() -now_iso = now.isoformat() +entry_template = textwrap.dedent(""" + <entry> + <id>https://yt-dl.org/feed/youtube-dl-updates-feed/youtube-dl-@VERSION@</id> + <title>New version @VERSION@</title> + <link href="http://rg3.github.io/youtube-dl" /> + <content type="xhtml"> + <div xmlns="http://www.w3.org/1999/xhtml"> + Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a> + </div> + </content> + <author> + <name>The youtube-dl maintainers</name> + </author> + <updated>@TIMESTAMP@</updated> + </entry> + """) -atom_template = atom_template.replace('@TIMESTAMP@',now_iso) +now = datetime.datetime.now() +now_iso = now.isoformat() + 'Z' -entries=[] +atom_template = atom_template.replace('@TIMESTAMP@', now_iso) versions_info = json.load(open('update/versions.json')) versions = list(versions_info['versions'].keys()) versions.sort() +entries = [] for v in versions: - entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-')) - entry = entry.replace('@VERSION@',v) - entries.append(entry) + fields = v.split('.') + year, month, day = map(int, fields[:3]) + faked = 0 + patchlevel = 0 + while True: + try: + datetime.date(year, month, day) + except ValueError: + day -= 1 + faked += 1 + assert day > 0 + continue + break + if len(fields) >= 4: + try: + patchlevel = int(fields[3]) + except ValueError: + patchlevel = 1 + timestamp = '%04d-%02d-%02dT00:%02d:%02dZ' % (year, month, day, faked, patchlevel) + + entry = entry_template.replace('@TIMESTAMP@', timestamp) + entry = entry.replace('@VERSION@', v) + entries.append(entry) entries_str = textwrap.indent(''.join(entries), '\t') atom_template = atom_template.replace('@ENTRIES@', entries_str) -with open('update/releases.atom','w',encoding='utf-8') as atom_file: - atom_file.write(atom_template) +with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: + atom_file.write(atom_template) diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 7f2ea31..cae1fa4 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -1,20 +1,24 @@ +import io import sys import re README_FILE = 'README.md' helptext = sys.stdin.read() -with open(README_FILE) as f: +if isinstance(helptext, bytes): + helptext = helptext.decode('utf-8') + +with io.open(README_FILE, encoding='utf-8') as f: oldreadme = f.read() header = oldreadme[:oldreadme.index('# OPTIONS')] footer = oldreadme[oldreadme.index('# CONFIGURATION'):] -options = helptext[helptext.index(' General Options:')+19:] +options = helptext[helptext.index(' General Options:') + 19:] options = re.sub(r'^ (\w.+)$', r'## \1', options, flags=re.M) options = '# OPTIONS\n' + options + '\n' -with open(README_FILE, 'w') as f: +with io.open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) f.write(options) f.write(footer) diff --git a/devscripts/release.sh b/devscripts/release.sh index 2766174..323acf8 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -24,6 +24,8 @@ if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.0 version="$1" if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi +useless_files=$(find youtube_dl -type f -not -name '*.py') +if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi /bin/echo -e "\n### First of all, testing..." |