LecygneNoir
/
prismedia
1
1
Fork 4
Code Issues 15 Pull Requests 0 Releases 22 Wiki Activity
Browse Source

Full rewrite of logging system to introduce --log as an option, and prepare the work for #29

pull/51/head
LecygneNoir 3 years ago
parent
25682af83a
commit
dc7dd5cb46
4 changed files with 114 additions and 74 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +29
    -27
      prismedia/pt_upload.py
  2. +52
    -20
      prismedia/upload.py
  3. +14
    -5
      prismedia/utils.py
  4. +19
    -22
      prismedia/yt_upload.py

+ 29
- 27
prismedia/pt_upload.py View File

@ -16,6 +16,7 @@ from oauthlib.oauth2 import LegacyApplicationClient
from requests_toolbelt.multipart.encoder import MultipartEncoder
from . import utils
logger = logging.getLogger('Prismedia')
PEERTUBE_SECRETS_FILE = 'peertube_secret'
PEERTUBE_PRIVACY = {
@ -43,10 +44,10 @@ def get_authenticated_service(secret):
)
except Exception as e:
if hasattr(e, 'message'):
logging.error("Peertube: Error: " + str(e.message))
logger.critical("Peertube: " + str(e.message))
exit(1)
else:
logging.error("Peertube: Error: " + str(e))
logger.critical("Peertube: " + str(e))
exit(1)
return oauth
@ -63,7 +64,7 @@ def get_channel_by_name(user_info, options):
def create_channel(oauth, url, options):
template = ('Peertube: Channel %s does not exist, creating it.')
logging.info(template % (str(options.get('--channel'))))
logger.info(template % (str(options.get('--channel'))))
channel_name = utils.cleanString(str(options.get('--channel')))
# Peertube allows 20 chars max for channel name
channel_name = channel_name[:19]
@ -81,23 +82,23 @@ def create_channel(oauth, url, options):
headers=headers)
except Exception as e:
if hasattr(e, 'message'):
logging.error("Error: " + str(e.message))
logger.error("Peertube: " + str(e.message))
else:
logging.error("Error: " + str(e))
logger.error("Peertube: " + str(e))
if response is not None:
if response.status_code == 200:
jresponse = response.json()
jresponse = jresponse['videoChannel']
return jresponse['id']
if response.status_code == 409:
logging.error('Peertube: Error: It seems there is a conflict with an existing channel named '
logger.critical('Peertube: It seems there is a conflict with an existing channel named '
+ channel_name + '.'
' Please beware Peertube internal name is compiled from 20 firsts characters of channel name.'
' Also note that channel name are not case sensitive (no uppercase nor accent)'
' Please check your channel name and retry.')
exit(1)
else:
logging.error(('Peertube: Creating channel failed with an unexpected response: '
logger.critical(('Peertube: Creating channel failed with an unexpected response: '
'%s') % response)
exit(1)
@ -114,7 +115,7 @@ def get_playlist_by_name(user_playlists, options):
def create_playlist(oauth, url, options, channel):
template = ('Peertube: Playlist %s does not exist, creating it.')
logging.info(template % (str(options.get('--playlist'))))
logger.info(template % (str(options.get('--playlist'))))
# We use files for form-data Content
# see https://requests.readthedocs.io/en/latest/user/quickstart/#post-a-multipart-encoded-file
# None is used to mute "filename" field
@ -128,22 +129,22 @@ def create_playlist(oauth, url, options, channel):
files=files)
except Exception as e:
if hasattr(e, 'message'):
logging.error("Error: " + str(e.message))
logger.error("Peertube: " + str(e.message))
else:
logging.error("Error: " + str(e))
logger.error("Peertube: " + str(e))
if response is not None:
if response.status_code == 200:
jresponse = response.json()
jresponse = jresponse['videoPlaylist']
return jresponse['id']
else:
logging.error(('Peertube: Creating the playlist failed with an unexpected response: '
logger.critical(('Peertube: Creating the playlist failed with an unexpected response: '
'%s') % response)
exit(1)
def set_playlist(oauth, url, video_id, playlist_id):
logging.info('Peertube: add video to playlist.')
logger.info('Peertube: add video to playlist.')
data = '{"videoId":"' + str(video_id) + '"}'
headers = {
@ -155,14 +156,14 @@ def set_playlist(oauth, url, video_id, playlist_id):
headers=headers)
except Exception as e:
if hasattr(e, 'message'):
logging.error("Error: " + str(e.message))
logger.error("Peertube: " + str(e.message))
else:
logging.error("Error: " + str(e))
logger.error("Peertube: " + str(e))
if response is not None:
if response.status_code == 200:
logging.info('Peertube: Video is successfully added to the playlist.')
logger.info('Peertube: Video is successfully added to the playlist.')
else:
logging.error(('Peertube: Configuring the playlist failed with an unexpected response: '
logger.critical(('Peertube: Configuring the playlist failed with an unexpected response: '
'%s') % response)
exit(1)
@ -205,8 +206,9 @@ def upload_video(oauth, secret, options):
continue
# Tag more than 30 chars crashes Peertube, so exit and check tags
if len(strtag) >= 30:
logging.warning("Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce tag: " + strtag)
exit(1)
logger.error("Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce tag: " + strtag)
logger.error("Peertube: Meanwhile, this tag will be skipped")
continue
fields.append(("tags[]", strtag))
if options.get('--category'):
@ -256,7 +258,7 @@ def upload_video(oauth, secret, options):
if not channel_id and options.get('--channelCreate'):
channel_id = create_channel(oauth, url, options)
elif not channel_id:
logging.warning("Channel `" + options.get('--channel') + "` is unknown, using default channel.")
logger.warning("Peertube: Channel `" + options.get('--channel') + "` is unknown, using default channel.")
channel_id = get_default_channel(user_info)
else:
channel_id = get_default_channel(user_info)
@ -268,7 +270,7 @@ def upload_video(oauth, secret, options):
if not playlist_id and options.get('--playlistCreate'):
playlist_id = create_playlist(oauth, url, options, channel_id)
elif not playlist_id:
logging.warning("Playlist `" + options.get('--playlist') + "` does not exist, please set --playlistCreate"
logger.critical("Peertube: Playlist `" + options.get('--playlist') + "` does not exist, please set --playlistCreate"
" if you want to create it")
exit(1)
@ -286,14 +288,14 @@ def upload_video(oauth, secret, options):
jresponse = jresponse['video']
uuid = jresponse['uuid']
video_id = str(jresponse['id'])
logging.info('Peertube : Video was successfully uploaded.')
logger.info('Peertube : Video was successfully uploaded.')
template = 'Peertube: Watch it at %s/videos/watch/%s.'
logging.info(template % (url, uuid))
logger.info(template % (url, uuid))
# Upload is successful we may set playlist
if options.get('--playlist'):
set_playlist(oauth, url, video_id, playlist_id)
else:
logging.error(('Peertube: The upload failed with an unexpected response: '
logger.critical(('Peertube: The upload failed with an unexpected response: '
'%s') % response)
exit(1)
@ -303,16 +305,16 @@ def run(options):
try:
secret.read(PEERTUBE_SECRETS_FILE)
except Exception as e:
logging.error("Peertube: Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e))
logger.critical("Peertube: Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e))
exit(1)
insecure_transport = secret.get('peertube', 'OAUTHLIB_INSECURE_TRANSPORT')
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = insecure_transport
oauth = get_authenticated_service(secret)
try:
logging.info('Peertube: Uploading video...')
logger.info('Peertube: Uploading video...')
upload_video(oauth, secret, options)
except Exception as e:
if hasattr(e, 'message'):
logging.error("Peertube: Error: " + str(e.message))
logger.error("Peertube: " + str(e.message))
else:
logging.error("Peertube: Error: " + str(e))
logger.error("Peertube: " + str(e))

+ 52
- 20
prismedia/upload.py View File

@ -13,7 +13,6 @@ Usage:
Options:
-f, --file=STRING Path to the video file to upload in mp4
--name=NAME Name of the video to upload. (default to video filename)
--debug Trigger some debug information like options used (default: no)
-d, --description=STRING Description of the video. (default: default description)
-t, --tags=STRING Tags for the video. comma separated.
WARN: tags with punctuation (!, ', ", ?, ...)
@ -46,6 +45,8 @@ Options:
If the playlist is not found, spawn an error except if --playlistCreate is set.
--playlistCreate Create the playlist if not exists. (default do not create)
Only relevant if --playlist is set.
--log=STRING Log level, between debug, info, warning, error, critical (default to info)
--debug (Deprecated) Alias for --log=DEBUG. Ignored if --log is set
-h --help Show this help.
--version Show version.
@ -92,7 +93,13 @@ if sys.version_info[0] < 3:
import os
import datetime
import logging
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
logger = logging.getLogger('Prismedia')
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s: %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
from docopt import docopt
@ -102,9 +109,9 @@ from . import utils
try:
# noinspection PyUnresolvedReferences
from schema import Schema, And, Or, Optional, SchemaError, Hook
from schema import Schema, And, Or, Optional, SchemaError, Hook, Use
except ImportError:
logging.error('This program requires that the `schema` data-validation library'
logger.critical('This program requires that the `schema` data-validation library'
' is installed: \n'
'see https://github.com/halst/schema\n')
exit(1)
@ -112,7 +119,7 @@ try:
# noinspection PyUnresolvedReferences
import magic
except ImportError:
logging.error('This program requires that the `python-magic` library'
logger.critical('This program requires that the `python-magic` library'
' is installed, NOT the Python bindings to libmagic API \n'
'see https://github.com/ahupp/python-magic\n')
exit(1)
@ -198,11 +205,17 @@ def validateThumbnail(thumbnail):
return False
def validateLogLevel(loglevel):
numeric_level = getattr(logging, loglevel, None)
if not isinstance(numeric_level, int):
return False
return True
def _optionnalOrStrict(key, scope, error):
option = key.replace('-', '')
option = option[0].upper() + option[1:]
if scope["--with" + option] is True and scope[key] is None:
logging.error("Prismedia: you have required the strict presence of " + key + " but none is found")
logger.critical("Prismedia: you have required the strict presence of " + key + " but none is found")
exit(1)
return True
@ -210,7 +223,13 @@ def _optionnalOrStrict(key, scope, error):
def main():
options = docopt(__doc__, version=VERSION)
strictoptionSchema = Schema({
earlyoptionSchema = Schema({
Optional('--log'): Or(None, And(
str,
Use(str.upper),
validateLogLevel,
error="Log level not recognized")
),
Optional('--withNFO', default=False): bool,
Optional('--withThumbnail', default=False): bool,
Optional('--withName', default=False): bool,
@ -222,7 +241,8 @@ def main():
Optional('--withCategory', default=False): bool,
Optional('--withLanguage', default=False): bool,
Optional('--withChannel', default=False): bool,
object: object # This allow to return all other options for further use: https://github.com/keleshev/schema#extra-keys
# This allow to return all other options for further use: https://github.com/keleshev/schema#extra-keys
object: object
})
schema = Schema({
@ -299,22 +319,34 @@ def main():
Optional('--playlistCreate'): bool,
'--help': bool,
'--version': bool,
object: object # This allow to return all other options for further use: https://github.com/keleshev/schema#extra-keys
# This allow to return all other options for further use: https://github.com/keleshev/schema#extra-keys
object: object
})
# We need to validate strict options first as withNFO should be validated before NFO parsing
# We need to validate early options first as withNFO and logs options should be prioritized
try:
options = strictoptionSchema.validate(options)
options = earlyoptionSchema.validate(options)
except SchemaError as e:
exit(e)
logger.critical(e)
exit(1)
if options.get('--log'):
numeric_level = getattr(logging, options["--log"], None)
# We need to set both log level in the same time
logger.setLevel(numeric_level)
ch.setLevel(numeric_level)
elif options.get('--debug'):
logger.warning("DEPRECATION: --debug is deprecated, please use --log=debug instead")
logger.setLevel(10)
ch.setLevel(10)
options = utils.parseNFO(options)
# Once NFO are loaded, we need to revalidate strict options in case some were in NFO
try:
options = strictoptionSchema.validate(options)
options = earlyoptionSchema.validate(options)
except SchemaError as e:
exit(e)
logger.critical(e)
exit(1)
if not options.get('--thumbnail'):
options = utils.searchThumbnail(options)
@ -322,11 +354,11 @@ def main():
try:
options = schema.validate(options)
except SchemaError as e:
exit(e)
logger.critical(e)
exit(1)
if options.get('--debug'):
print("Python " + sys.version)
print(options)
logger.debug("Python " + sys.version)
logger.debug(options)
if options.get('--platform') is None or "peertube" in options.get('--platform'):
pt_upload.run(options)
@ -336,5 +368,5 @@ def main():
if __name__ == '__main__':
import warnings
warnings.warn("use 'python -m prismedia', not 'python -m prismedia.upload'", DeprecationWarning)
logger.warning("DEPRECATION: use 'python -m prismedia', not 'python -m prismedia.upload'")
main()

+ 14
- 5
prismedia/utils.py View File

@ -9,6 +9,8 @@ from subprocess import check_call, CalledProcessError, STDOUT
import unidecode
import logging
logger = logging.getLogger('Prismedia')
### CATEGORIES ###
YOUTUBE_CATEGORY = {
"music": 10,
@ -123,18 +125,25 @@ def searchThumbnail(options):
options['--thumbnail'] = video_directory + video_file + ".jpg"
elif isfile(video_directory + video_file + ".jpeg"):
options['--thumbnail'] = video_directory + video_file + ".jpeg"
# Display some info after research
if not options.get('--thumbnail'):
logger.debug("No thumbnail has been found, continuing")
else:
logger.info("Using " + options.get('--thumbnail') + "as thumbnail")
return options
# return the nfo as a RawConfigParser object
def loadNFO(filename):
try:
logging.info("Loading " + filename + " as NFO")
logger.info("Loading " + filename + " as NFO")
nfo = RawConfigParser()
nfo.read(filename, encoding='utf-8')
return nfo
except Exception as e:
logging.error("Problem loading NFO file " + filename + ": " + str(e))
logger.critical("Problem loading NFO file " + filename + ": " + str(e))
exit(1)
return False
@ -168,7 +177,7 @@ def parseNFO(options):
if isfile(options.get('--nfo')):
nfo_cli = loadNFO(options.get('--nfo'))
else:
logging.error("Given NFO file does not exist, please check your path.")
logger.critical("Given NFO file does not exist, please check your path.")
exit(1)
# If there is no NFO and strict option is enabled, then stop there
@ -178,7 +187,7 @@ def parseNFO(options):
not isinstance(nfo_videoname, RawConfigParser) and \
not isinstance(nfo_directory, RawConfigParser) and \
not isinstance(nfo_txt, RawConfigParser):
logging.error("Prismedia: you have required the strict presence of NFO but none is found, please use a NFO.")
logger.critical("You have required the strict presence of NFO but none is found, please use a NFO.")
exit(1)
# We need to load NFO in this exact order to keep the priorities
@ -198,7 +207,7 @@ def parseNFO(options):
except NoOptionError:
continue
except NoSectionError:
logging.error("Prismedia: " + nfo + " misses section [video], please check syntax of your NFO.")
logger.critical(nfo + " misses section [video], please check syntax of your NFO.")
exit(1)
return options

+ 19
- 22
prismedia/yt_upload.py View File

@ -23,9 +23,7 @@ from google_auth_oauthlib.flow import InstalledAppFlow
from . import utils
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
logger = logging.getLogger('Prismedia')
# Explicitly tell the underlying HTTP transport library not to retry, since
# we are handling retry logic ourselves.
@ -87,7 +85,7 @@ def check_authenticated_scopes():
credential_params = json.load(f)
# Check if all scopes are present
if credential_params["_scopes"] != SCOPES:
logging.warning("Youtube: Credentials are obsolete, need to re-authenticate.")
logger.warning("Youtube: Credentials are obsolete, need to re-authenticate.")
os.remove(CREDENTIALS_PATH)
@ -144,8 +142,8 @@ def initialize_upload(youtube, options):
if not playlist_id and options.get('--playlistCreate'):
playlist_id = create_playlist(youtube, options.get('--playlist'))
elif not playlist_id:
logging.warning("Youtube: Playlist `" + options.get('--playlist') + "` is unknown.")
logging.warning("If you want to create it, set the --playlistCreate option.")
logger.warning("Youtube: Playlist `" + options.get('--playlist') + "` is unknown.")
logger.warning("Youtube: If you want to create it, set the --playlistCreate option.")
playlist_id = ""
else:
playlist_id = ""
@ -179,8 +177,8 @@ def get_playlist_by_name(youtube, playlist_name):
def create_playlist(youtube, playlist_name):
template = ('Youtube: Playlist %s does not exist, creating it.')
logging.info(template % (str(playlist_name)))
template = 'Youtube: Playlist %s does not exist, creating it.'
logger.info(template % (str(playlist_name)))
resources = build_resource({'snippet.title': playlist_name,
'snippet.description': '',
'status.privacyStatus': 'public'})
@ -244,7 +242,7 @@ def set_thumbnail(youtube, media_file, **kwargs):
def set_playlist(youtube, playlist_id, video_id):
logging.info('Youtube: Configuring playlist...')
logger.info('Youtube: Configuring playlist...')
resource = build_resource({'snippet.playlistId': playlist_id,
'snippet.resourceId.kind': 'youtube#video',
'snippet.resourceId.videoId': video_id,
@ -257,12 +255,12 @@ def set_playlist(youtube, playlist_id, video_id):
).execute()
except Exception as e:
if hasattr(e, 'message'):
logging.error("Youtube: Error: " + str(e.message))
logger.critical("Youtube: " + str(e.message))
exit(1)
else:
logging.error("Youtube: Error: " + str(e))
logger.critical("Youtube: " + str(e))
exit(1)
logging.info('Youtube: Video is correctly added to the playlist.')
logger.info('Youtube: Video is correctly added to the playlist.')
# This method implements an exponential backoff strategy to resume a
@ -274,20 +272,20 @@ def resumable_upload(request, resource, method):
while response is None:
try:
template = 'Youtube: Uploading %s...'
logging.info(template % resource)
logger.info(template % resource)
status, response = request.next_chunk()
if response is not None:
if method == 'insert' and 'id' in response:
logging.info('Youtube : Video was successfully uploaded.')
logger.info('Youtube : Video was successfully uploaded.')
template = 'Youtube: Watch it at https://youtu.be/%s (post-encoding could take some time)'
logging.info(template % response['id'])
logger.info(template % response['id'])
return response['id']
elif method != 'insert' or "id" not in response:
logging.info('Youtube: Thumbnail was successfully set.')
logger.info('Youtube: Thumbnail was successfully set.')
else:
template = ('Youtube : The upload failed with an '
'unexpected response: %s')
logging.error(template % response)
logger.critical(template % response)
exit(1)
except HttpError as e:
if e.resp.status in RETRIABLE_STATUS_CODES:
@ -299,15 +297,14 @@ def resumable_upload(request, resource, method):
error = 'Youtube : A retriable error occurred: %s' % e
if error is not None:
logging.warning(error)
logger.warning(error)
retry += 1
if retry > MAX_RETRIES:
logging.error('Youtube : No longer attempting to retry.')
exit(1)
logger.error('Youtube : No longer attempting to retry.')
max_sleep = 2 ** retry
sleep_seconds = random.random() * max_sleep
logging.warning('Youtube : Sleeping %f seconds and then retrying...'
logger.warning('Youtube : Sleeping %f seconds and then retrying...'
% sleep_seconds)
time.sleep(sleep_seconds)
@ -317,5 +314,5 @@ def run(options):
try:
initialize_upload(youtube, options)
except HttpError as e:
logging.error('Youtube : An HTTP error %d occurred:\n%s' % (e.resp.status,
logger.error('Youtube : An HTTP error %d occurred:\n%s' % (e.resp.status,
e.content))

Write Preview
Loading…
Cancel
Save