From e8cd8c4bd832446f1971215b9fedc4531555dc1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Brito?= Date: Sun, 19 Jan 2014 02:38:23 -0200 Subject: Imported Upstream version 2014.01.17.2 --- devscripts/bash-completion.in | 2 +- devscripts/check-porn.py | 44 +++++++++++++----- devscripts/gh-pages/update-feed.py | 92 +++++++++++++++++++++++--------------- devscripts/make_readme.py | 10 +++-- devscripts/release.sh | 2 + 5 files changed, 98 insertions(+), 52 deletions(-) (limited to 'devscripts') diff --git a/devscripts/bash-completion.in b/devscripts/bash-completion.in index 3af87a3..28bd237 100644 --- a/devscripts/bash-completion.in +++ b/devscripts/bash-completion.in @@ -6,7 +6,7 @@ __youtube_dl() prev="${COMP_WORDS[COMP_CWORD-1]}" opts="{{flags}}" keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" - fileopts="-a|--batch-file|--download-archive|--cookies" + fileopts="-a|--batch-file|--download-archive|--cookies|--load-info" diropts="--cache-dir" if [[ ${prev} =~ ${fileopts} ]]; then diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 63401fe..86aa37b 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -3,6 +3,9 @@ """ This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check if we are not 'age_limit' tagging some porn site + +A second approach implemented relies on a list of porn domains, to activate it +pass the list filename as the only argument """ # Allow direct execution @@ -11,25 +14,42 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import get_testcases +from youtube_dl.utils import compat_urllib_parse_urlparse from youtube_dl.utils import compat_urllib_request +if len(sys.argv) > 1: + METHOD = 'LIST' + LIST = open(sys.argv[1]).read().decode('utf8').strip() +else: + METHOD = 'EURISTIC' + for test in get_testcases(): - try: - webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() - except: - print('\nFail: {0}'.format(test['name'])) - continue + if METHOD == 'EURISTIC': + try: + webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() + except: + print('\nFail: {0}'.format(test['name'])) + continue + + webpage = webpage.decode('utf8', 'replace') + + RESULT = 'porn' in webpage.lower() + + elif METHOD == 'LIST': + domain = compat_urllib_parse_urlparse(test['url']).netloc + if not domain: + print('\nFail: {0}'.format(test['name'])) + continue + domain = '.'.join(domain.split('.')[-2:]) - webpage = webpage.decode('utf8', 'replace') + RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST) - if 'porn' in webpage.lower() and ('info_dict' not in test - or 'age_limit' not in test['info_dict'] - or test['info_dict']['age_limit'] != 18): + if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] + or test['info_dict']['age_limit'] != 18): print('\nPotential missing age_limit check: {0}'.format(test['name'])) - elif 'porn' not in webpage.lower() and ('info_dict' in test and - 'age_limit' in test['info_dict'] and - test['info_dict']['age_limit'] == 18): + elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] + and test['info_dict']['age_limit'] == 18): print('\nPotential false negative: {0}'.format(test['name'])) else: diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py index 16571a9..0ba15ae 100755 --- a/devscripts/gh-pages/update-feed.py +++ b/devscripts/gh-pages/update-feed.py @@ -1,56 +1,76 @@ #!/usr/bin/env python3 import datetime - +import io +import json import textwrap -import json -atom_template=textwrap.dedent("""\ - - - youtube-dl releases - youtube-dl-updates-feed - @TIMESTAMP@ - @ENTRIES@ - """) - -entry_template=textwrap.dedent(""" - - youtube-dl-@VERSION@ - New version @VERSION@ - - -
- Downloads available at https://yt-dl.org/downloads/@VERSION@/ -
- - - The youtube-dl maintainers - - @TIMESTAMP@ - - """) +atom_template = textwrap.dedent("""\ + + + + youtube-dl releases + https://yt-dl.org/feed/youtube-dl-updates-feed + @TIMESTAMP@ + @ENTRIES@ + """) -now = datetime.datetime.now() -now_iso = now.isoformat() +entry_template = textwrap.dedent(""" + + https://yt-dl.org/feed/youtube-dl-updates-feed/youtube-dl-@VERSION@ + New version @VERSION@ + + +
+ Downloads available at https://yt-dl.org/downloads/@VERSION@/ +
+ + + The youtube-dl maintainers + + @TIMESTAMP@ + + """) -atom_template = atom_template.replace('@TIMESTAMP@',now_iso) +now = datetime.datetime.now() +now_iso = now.isoformat() + 'Z' -entries=[] +atom_template = atom_template.replace('@TIMESTAMP@', now_iso) versions_info = json.load(open('update/versions.json')) versions = list(versions_info['versions'].keys()) versions.sort() +entries = [] for v in versions: - entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-')) - entry = entry.replace('@VERSION@',v) - entries.append(entry) + fields = v.split('.') + year, month, day = map(int, fields[:3]) + faked = 0 + patchlevel = 0 + while True: + try: + datetime.date(year, month, day) + except ValueError: + day -= 1 + faked += 1 + assert day > 0 + continue + break + if len(fields) >= 4: + try: + patchlevel = int(fields[3]) + except ValueError: + patchlevel = 1 + timestamp = '%04d-%02d-%02dT00:%02d:%02dZ' % (year, month, day, faked, patchlevel) + + entry = entry_template.replace('@TIMESTAMP@', timestamp) + entry = entry.replace('@VERSION@', v) + entries.append(entry) entries_str = textwrap.indent(''.join(entries), '\t') atom_template = atom_template.replace('@ENTRIES@', entries_str) -with open('update/releases.atom','w',encoding='utf-8') as atom_file: - atom_file.write(atom_template) +with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: + atom_file.write(atom_template) diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 7f2ea31..cae1fa4 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -1,20 +1,24 @@ +import io import sys import re README_FILE = 'README.md' helptext = sys.stdin.read() -with open(README_FILE) as f: +if isinstance(helptext, bytes): + helptext = helptext.decode('utf-8') + +with io.open(README_FILE, encoding='utf-8') as f: oldreadme = f.read() header = oldreadme[:oldreadme.index('# OPTIONS')] footer = oldreadme[oldreadme.index('# CONFIGURATION'):] -options = helptext[helptext.index(' General Options:')+19:] +options = helptext[helptext.index(' General Options:') + 19:] options = re.sub(r'^ (\w.+)$', r'## \1', options, flags=re.M) options = '# OPTIONS\n' + options + '\n' -with open(README_FILE, 'w') as f: +with io.open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) f.write(options) f.write(footer) diff --git a/devscripts/release.sh b/devscripts/release.sh index 2766174..323acf8 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -24,6 +24,8 @@ if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.0 version="$1" if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi +useless_files=$(find youtube_dl -type f -not -name '*.py') +if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi /bin/echo -e "\n### First of all, testing..." -- cgit v1.1