diff --git a/PLUGINS.md b/PLUGINS.md new file mode 100644 index 0000000..36f3374 --- /dev/null +++ b/PLUGINS.md @@ -0,0 +1,13 @@ +For our plugins we are using [yaspy](http://yapsy.sourceforge.net). + +# Types +For an example of the exact methods required to be recognized as a particular type of plugin, see the concerned interface definition. + +## Interface +Plugins that present an interface (cli, gui, configuration folders, …) for the user to tell Prismedia wich video needs to be uploaded, the infos of the videos, … + +## Platform +Also called uploaders, they are the one doing the actual work of uploading video to a particular platform. + +## Consumer +Thoses do actions once the upload is finished (successful or failed). diff --git a/prismedia/core.py b/prismedia/core.py new file mode 100644 index 0000000..0604666 --- /dev/null +++ b/prismedia/core.py @@ -0,0 +1,62 @@ +from yapsy.PluginManager import PluginManager +import pluginInterfaces as pi +import logging +# logging.basicConfig(level=logging.DEBUG) + +def loadPlugins(type): + # Load the plugins from the plugin directory. + # TODO: subdirectories too? + manager = PluginManager() + manager.setPluginPlaces(["plugins"]) # TODO: Generate the absolute path + + # Define the various categories corresponding to the different + # kinds of plugins you have defined + manager.setCategoriesFilter({ + "Interface" : pi.IInterfacePlugin, + "Platform" : pi.IPlatformPlugin, + }) + + manager.collectPlugins() + + # Loop round the plugins and print their names. + print("debug") + print(manager.getAllPlugins()) + + print("all plugins") + for plugin in manager.getAllPlugins(): + plugin.plugin_object.print_name() + + print("Category: Interface") + for plugin in manager.getPluginsOfCategory("Interface"): + plugin.plugin_object.print_name() + + print("Category: Platform") + for plugin in manager.getPluginsOfCategory("Platform"): + plugin.plugin_object.print_name() + + # discovered_plugins = { + # name: importlib.import_module(name) + # for finder, name, ispkg + # in pkgutil.iter_modules(["/home/zykino/Documents/0DocPerso/Code/prismedia/plugins"]) + # if name.startswith("prismedia_" + type + "_") + # } + +#def test_loadPlugins(arg): +platforms = loadPlugins("platform") +print (platforms) + +def startInterface(): + interface = loadPlugins("interface") + + options = interface["default"].run() + if options.get('--interface'): + if interface[options.get('--interface')]: + options = interface[options.get('--interface')].run(options) + else: + options = interface["cli"].run(options) + options = interface["nfo"].run(options) + +def uploadToPlatforms(options): + platforms = loadPlugins("platform") + for platform in options.get('--platform'): + platforms[platform].run(options) diff --git a/prismedia/pluginInterfaces.py b/prismedia/pluginInterfaces.py new file mode 100644 index 0000000..2f8fd5d --- /dev/null +++ b/prismedia/pluginInterfaces.py @@ -0,0 +1,60 @@ +from yapsy.IPlugin import IPlugin + +### +# Interface +### +# TODO: The interface is not thought out yet +class IInterfacePlugin(IPlugin): + """ + Interface for the Interface plugin category. + """ + + def getOptions(self, args): + """ + Returns the options user has set. + - `args` the command line arguments passed to Prismedia + """ + raise NotImplementedError("`getOptions` must be reimplemented by %s" % self) + + +### +# Platform +### +class IPlatformPlugin(IPlugin): + """ + Interface for the Platform plugin category. + """ + + # def dryrun(self, video, options): + # """ + # Simulate an upload but without really uploading anything. + # """ + # raise NotImplementedError("`dryrun` must be reimplemented by %s" % self) + + def hearthbeat(self): + """ + If needed for your platform, use a bit of the api so the platform is aware the keys are still in use. + """ + raise NotImplementedError("`hearthbeat` must be reimplemented by %s" % self) + + def upload(self, video, options): + """ + The upload function + """ + raise NotImplementedError("`upload` must be reimplemented by %s" % self) + +### +# Consumer +### +# TODO: The interface is not thought out yet +class IConsumerPlugin(IPlugin): + """ + Interface for the Consumer plugin category. + """ + + def finished(self, video): + """ + What to do once the uploads are done. + - `video` is an object containing the video details. The `platforms` key contain a list of the platforms the video has been uploaded to and the status + """ + raise NotImplementedError("`getOptions` must be reimplemented by %s" % self) diff --git a/prismedia/plugins/platforms/peertube.py b/prismedia/plugins/platforms/peertube.py new file mode 100644 index 0000000..78780a2 --- /dev/null +++ b/prismedia/plugins/platforms/peertube.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python +# coding: utf-8 + +import os +import mimetypes +import json +import logging +import sys +import datetime +import pytz +import pluginInterfaces as pi +from os.path import splitext, basename, abspath +from tzlocal import get_localzone + +from configparser import RawConfigParser +from requests_oauthlib import OAuth2Session +from oauthlib.oauth2 import LegacyApplicationClient +from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor +from clint.textui.progress import Bar as ProgressBar + +import utils +logger = logging.getLogger('Prismedia') + +PEERTUBE_SECRETS_FILE = 'peertube_secret' +PEERTUBE_PRIVACY = { + "public": 1, + "unlisted": 2, + "private": 3 +} + +CATEGORY = { + "music": 1, + "films": 2, + "vehicles": 3, + "sport": 5, + "travels": 6, + "gaming": 7, + "people": 8, + "comedy": 9, + "entertainment": 10, + "news": 11, + "how to": 12, + "education": 13, + "activism": 14, + "science & technology": 15, + "science": 15, + "technology": 15, + "animals": 16 +} + +LANGUAGE = { + "arabic": "ar", + "english": "en", + "french": "fr", + "german": "de", + "hindi": "hi", + "italian": "it", + "japanese": "ja", + "korean": "ko", + "mandarin": "zh", + "portuguese": "pt", + "punjabi": "pa", + "russian": "ru", + "spanish": "es" +} + +class Peertube(pi.IPlatformPlugin): + """docstring for Peertube.""" + def print_name(self): + print("This is plugin peertube") + + def get_authenticated_service(secret): + peertube_url = str(secret.get('peertube', 'peertube_url')).rstrip("/") + + oauth_client = LegacyApplicationClient( + client_id=str(secret.get('peertube', 'client_id')) + ) + try: + oauth = OAuth2Session(client=oauth_client) + oauth.fetch_token( + token_url=str(peertube_url + '/api/v1/users/token'), + # lower as peertube does not store uppercase for pseudo + username=str(secret.get('peertube', 'username').lower()), + password=str(secret.get('peertube', 'password')), + client_id=str(secret.get('peertube', 'client_id')), + client_secret=str(secret.get('peertube', 'client_secret')) + ) + except Exception as e: + if hasattr(e, 'message'): + logger.critical("Peertube: " + str(e.message)) + exit(1) + else: + logger.critical("Peertube: " + str(e)) + exit(1) + return oauth + + + def get_default_channel(user_info): + return user_info['videoChannels'][0]['id'] + + + def get_channel_by_name(user_info, options): + for channel in user_info["videoChannels"]: + if channel['displayName'] == options.get('--channel'): + return channel['id'] + + + def convert_peertube_date(date): + date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S') + tz = get_localzone() + tz = pytz.timezone(str(tz)) + return tz.localize(date).isoformat() + + + def create_channel(oauth, url, options): + template = ('Peertube: Channel %s does not exist, creating it.') + logger.info(template % (str(options.get('--channel')))) + channel_name = utils.cleanString(str(options.get('--channel'))) + # Peertube allows 20 chars max for channel name + channel_name = channel_name[:19] + data = '{"name":"' + channel_name + '", \ + "displayName":"' + options.get('--channel') + '", \ + "description":null, \ + "support":null}' + + headers = { + 'Content-Type': "application/json; charset=UTF-8" + } + try: + response = oauth.post(url + "/api/v1/video-channels/", + data=data.encode('utf-8'), + headers=headers) + except Exception as e: + if hasattr(e, 'message'): + logger.error("Peertube: " + str(e.message)) + else: + logger.error("Peertube: " + str(e)) + if response is not None: + if response.status_code == 200: + jresponse = response.json() + jresponse = jresponse['videoChannel'] + return jresponse['id'] + if response.status_code == 409: + logger.critical('Peertube: It seems there is a conflict with an existing channel named ' + + channel_name + '.' + ' Please beware Peertube internal name is compiled from 20 firsts characters of channel name.' + ' Also note that channel name are not case sensitive (no uppercase nor accent)' + ' Please check your channel name and retry.') + exit(1) + else: + logger.critical(('Peertube: Creating channel failed with an unexpected response: ' + '%s') % response) + exit(1) + + + def get_default_playlist(user_info): + return user_info['videoChannels'][0]['id'] + + + def get_playlist_by_name(oauth, url, username, options): + start = 0 + user_playlists = json.loads(oauth.get( + url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content) + total = user_playlists["total"] + data = user_playlists["data"] + # We need to iterate on pagination as peertube returns max 100 playlists (see #41) + while start < total: + for playlist in data: + if playlist['displayName'] == options.get('--playlist'): + return playlist['id'] + start = start + 100 + user_playlists = json.loads(oauth.get( + url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content) + data = user_playlists["data"] + + + def create_playlist(oauth, url, options, channel): + template = ('Peertube: Playlist %s does not exist, creating it.') + logger.info(template % (str(options.get('--playlist')))) + # We use files for form-data Content + # see https://requests.readthedocs.io/en/latest/user/quickstart/#post-a-multipart-encoded-file + # None is used to mute "filename" field + files = {'displayName': (None, str(options.get('--playlist'))), + 'privacy': (None, "1"), + 'description': (None, "null"), + 'videoChannelId': (None, str(channel)), + 'thumbnailfile': (None, "null")} + try: + response = oauth.post(url + "/api/v1/video-playlists/", + files=files) + except Exception as e: + if hasattr(e, 'message'): + logger.error("Peertube: " + str(e.message)) + else: + logger.error("Peertube: " + str(e)) + if response is not None: + if response.status_code == 200: + jresponse = response.json() + jresponse = jresponse['videoPlaylist'] + return jresponse['id'] + else: + logger.critical(('Peertube: Creating the playlist failed with an unexpected response: ' + '%s') % response) + exit(1) + + + def set_playlist(oauth, url, video_id, playlist_id): + logger.info('Peertube: add video to playlist.') + data = '{"videoId":"' + str(video_id) + '"}' + + headers = { + 'Content-Type': "application/json" + } + try: + response = oauth.post(url + "/api/v1/video-playlists/"+str(playlist_id)+"/videos", + data=data, + headers=headers) + except Exception as e: + if hasattr(e, 'message'): + logger.error("Peertube: " + str(e.message)) + else: + logger.error("Peertube: " + str(e)) + if response is not None: + if response.status_code == 200: + logger.info('Peertube: Video is successfully added to the playlist.') + else: + logger.critical(('Peertube: Configuring the playlist failed with an unexpected response: ' + '%s') % response) + exit(1) + + + def upload_video(oauth, secret, options): + + def get_userinfo(): + return json.loads(oauth.get(url+"/api/v1/users/me").content) + + def get_file(path): + mimetypes.init() + return (basename(path), open(abspath(path), 'rb'), + mimetypes.types_map[splitext(path)[1]]) + + path = options.get('--file') + url = str(secret.get('peertube', 'peertube_url')).rstrip('/') + user_info = get_userinfo() + username = str(secret.get('peertube', 'username').lower()) + + # We need to transform fields into tuple to deal with tags as + # MultipartEncoder does not support list refer + # https://github.com/requests/toolbelt/issues/190 and + # https://github.com/requests/toolbelt/issues/205 + fields = [ + ("name", options.get('--name') or splitext(basename(options.get('--file')))[0]), + ("licence", "1"), + ("description", options.get('--description') or "default description"), + ("nsfw", str(int(options.get('--nsfw')) or "0")), + ("videofile", get_file(path)) + ] + + if options.get('--tags'): + tags = options.get('--tags').split(',') + tag_number = 0 + for strtag in tags: + tag_number = tag_number + 1 + # Empty tag crashes Peertube, so skip them + if strtag == "": + continue + # Tag more than 30 chars crashes Peertube, so skip tags + if len(strtag) >= 30: + logger.warning("Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce tag: " + strtag) + logger.warning("Peertube: Meanwhile, this tag will be skipped") + continue + # Peertube supports only 5 tags at the moment + if tag_number > 5: + logger.warning("Peertube: Sorry, Peertube support 5 tags max, additional tag will be skipped") + logger.warning("Peertube: Skipping tag " + strtag) + continue + fields.append(("tags[]", strtag)) + + if options.get('--category'): + fields.append(("category", str(CATEGORY[options.get('--category').lower()]))) + else: + # if no category, set default to 2 (Films) + fields.append(("category", "2")) + + if options.get('--language'): + fields.append(("language", str(LANGUAGE[options.get('--language').lower()]))) + else: + # if no language, set default to 1 (English) + fields.append(("language", "en")) + + if options.get('--disable-comments'): + fields.append(("commentsEnabled", "0")) + else: + fields.append(("commentsEnabled", "1")) + + privacy = None + if options.get('--privacy'): + privacy = options.get('--privacy').lower() + + # If peertubeAt exists, use instead of publishAt + if options.get('--peertubeAt'): + publishAt = options.get('--peertubeAt') + elif options.get('--publishAt'): + publishAt = options.get('--publishAt') + + if 'publishAt' in locals(): + publishAt = convert_peertube_date(publishAt) + fields.append(("scheduleUpdate[updateAt]", publishAt)) + fields.append(("scheduleUpdate[privacy]", str(PEERTUBE_PRIVACY["public"]))) + fields.append(("privacy", str(PEERTUBE_PRIVACY["private"]))) + else: + fields.append(("privacy", str(PEERTUBE_PRIVACY[privacy or "private"]))) + + # Set originalDate except if the user force no originalDate + if options.get('--originalDate'): + originalDate = convert_peertube_date(options.get('--originalDate')) + fields.append(("originallyPublishedAt", originalDate)) + + if options.get('--thumbnail'): + fields.append(("thumbnailfile", get_file(options.get('--thumbnail')))) + fields.append(("previewfile", get_file(options.get('--thumbnail')))) + + if options.get('--channel'): + channel_id = get_channel_by_name(user_info, options) + if not channel_id and options.get('--channelCreate'): + channel_id = create_channel(oauth, url, options) + elif not channel_id: + logger.warning("Peertube: Channel `" + options.get('--channel') + "` is unknown, using default channel.") + channel_id = get_default_channel(user_info) + else: + channel_id = get_default_channel(user_info) + + fields.append(("channelId", str(channel_id))) + + if options.get('--playlist'): + playlist_id = get_playlist_by_name(oauth, url, username, options) + if not playlist_id and options.get('--playlistCreate'): + playlist_id = create_playlist(oauth, url, options, channel_id) + elif not playlist_id: + logger.critical("Peertube: Playlist `" + options.get('--playlist') + "` does not exist, please set --playlistCreate" + " if you want to create it") + exit(1) + + logger_stdout = None + if options.get('--url-only') or options.get('--batch'): + logger_stdout = logging.getLogger('stdoutlogs') + + encoder = MultipartEncoder(fields) + if options.get('--quiet'): + multipart_data = encoder + else: + progress_callback = create_callback(encoder, options.get('--progress')) + multipart_data = MultipartEncoderMonitor(encoder, progress_callback) + + headers = { + 'Content-Type': multipart_data.content_type + } + response = oauth.post(url + "/api/v1/videos/upload", + data=multipart_data, + headers=headers) + + if response is not None: + if response.status_code == 200: + jresponse = response.json() + jresponse = jresponse['video'] + uuid = jresponse['uuid'] + video_id = str(jresponse['id']) + + logger.info('Peertube: Video was successfully uploaded.') + template = 'Peertube: Watch it at %s/videos/watch/%s.' + logger.info(template % (url, uuid)) + template_stdout = '%s/videos/watch/%s' + if options.get('--url-only'): + logger_stdout.info(template_stdout % (url, uuid)) + elif options.get('--batch'): + logger_stdout.info("Peertube: " + template_stdout % (url, uuid)) + # Upload is successful we may set playlist + if options.get('--playlist'): + set_playlist(oauth, url, video_id, playlist_id) + else: + logger.critical(('Peertube: The upload failed with an unexpected response: ' + '%s') % response) + exit(1) + + + upload_finished = False + def create_callback(encoder, progress_type): + upload_size_MB = encoder.len * (1 / (1024 * 1024)) + + if progress_type is None or "percentage" in progress_type.lower(): + progress_lambda = lambda x: int((x / encoder.len) * 100) # Default to percentage + elif "bigfile" in progress_type.lower(): + progress_lambda = lambda x: x * (1 / (1024 * 1024)) # MB + elif "accurate" in progress_type.lower(): + progress_lambda = lambda x: x * (1 / (1024)) # kB + else: + # Should not happen outside of development when adding partly a progress type + logger.critical("Peertube: Unknown progress type `" + progress_type + "`") + exit(1) + + bar = ProgressBar(expected_size=progress_lambda(encoder.len), label=f"Peertube upload progress ({upload_size_MB:.2f}MB) ", filled_char='=') + + def callback(monitor): + # We want the condition to capture the varible from the parent scope, not a local variable that is created after + global upload_finished + progress = progress_lambda(monitor.bytes_read) + + bar.show(progress) + + if monitor.bytes_read == encoder.len: + if not upload_finished: + # We get two time in the callback with both bytes equals, skip the first + upload_finished = True + else: + # Print a blank line to not (partly) override the progress bar + print() + logger.info("Peertube: Upload finish, Processing…") + + return callback + + + def hearthbeat(self): + """ + If needed for your platform, use a bit of the api so the platform is aware the keys are still in use. + """ + pass + + + def upload(self, video, options): + # def run(options): + secret = RawConfigParser() + try: + secret.read(PEERTUBE_SECRETS_FILE) + except Exception as e: + logger.critical("Peertube: Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e)) + exit(1) + insecure_transport = secret.get('peertube', 'OAUTHLIB_INSECURE_TRANSPORT') + os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = insecure_transport + oauth = get_authenticated_service(secret) + try: + logger.info('Peertube: Uploading video...') + upload_video(oauth, secret, options) + except Exception as e: + if hasattr(e, 'message'): + logger.error("Peertube: " + str(e.message)) + else: + logger.error("Peertube: " + str(e)) diff --git a/prismedia/plugins/platforms/peertube.yapsy-plugin b/prismedia/plugins/platforms/peertube.yapsy-plugin new file mode 100644 index 0000000..a07437d --- /dev/null +++ b/prismedia/plugins/platforms/peertube.yapsy-plugin @@ -0,0 +1,9 @@ +[Core] +Name = Peertube +Module = peertube + +[Documentation] +Author = Le Cygne Noir +Version = 0.1 +Website = https://git.lecygnenoir.info/LecygneNoir/prismedia +Description = Upload to the peertube platform diff --git a/prismedia/yt_upload.py b/prismedia/plugins/platforms/youtube.py similarity index 94% rename from prismedia/yt_upload.py rename to prismedia/plugins/platforms/youtube.py index e8809c5..0faeb95 100644 --- a/prismedia/yt_upload.py +++ b/prismedia/plugins/platforms/youtube.py @@ -54,6 +54,41 @@ SCOPES = ['https://www.googleapis.com/auth/youtube.upload', 'https://www.googlea API_SERVICE_NAME = 'youtube' API_VERSION = 'v3' +CATEGORY = { + "music": 10, + "films": 1, + "vehicles": 2, + "sport": 17, + "travels": 19, + "gaming": 20, + "people": 22, + "comedy": 23, + "entertainment": 24, + "news": 25, + "how to": 26, + "education": 27, + "activism": 29, + "science & technology": 28, + "science": 28, + "technology": 28, + "animals": 15 +} + +LANGUAGE = { + "arabic": 'ar', + "english": 'en', + "french": 'fr', + "german": 'de', + "hindi": 'hi', + "italian": 'it', + "japanese": 'ja', + "korean": 'ko', + "mandarin": 'zh-CN', + "portuguese": 'pt-PT', + "punjabi": 'pa', + "russian": 'ru', + "spanish": 'es' +} # Authorize the request and store authorization credentials. def get_authenticated_service(): @@ -107,11 +142,11 @@ def initialize_upload(youtube, options): category = None if options.get('--category'): - category = utils.getCategory(options.get('--category'), 'youtube') + category = CATEGORY[options.get('--category').lower()] language = None if options.get('--language'): - language = utils.getLanguage(options.get('--language'), "youtube") + language = LANGUAGE[options.get('--language').lower()] license = None if options.get('--cca'): diff --git a/prismedia/pt_upload.py b/prismedia/pt_upload.py deleted file mode 100644 index adb5a36..0000000 --- a/prismedia/pt_upload.py +++ /dev/null @@ -1,398 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -import mimetypes -import json -import logging -import sys -import datetime -import pytz -from os.path import splitext, basename, abspath -from tzlocal import get_localzone - -from configparser import RawConfigParser -from requests_oauthlib import OAuth2Session -from oauthlib.oauth2 import LegacyApplicationClient -from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor -from clint.textui.progress import Bar as ProgressBar - -from . import utils -logger = logging.getLogger('Prismedia') - -PEERTUBE_SECRETS_FILE = 'peertube_secret' -PEERTUBE_PRIVACY = { - "public": 1, - "unlisted": 2, - "private": 3 -} - - -def get_authenticated_service(secret): - peertube_url = str(secret.get('peertube', 'peertube_url')).rstrip("/") - - oauth_client = LegacyApplicationClient( - client_id=str(secret.get('peertube', 'client_id')) - ) - try: - oauth = OAuth2Session(client=oauth_client) - oauth.fetch_token( - token_url=str(peertube_url + '/api/v1/users/token'), - # lower as peertube does not store uppercase for pseudo - username=str(secret.get('peertube', 'username').lower()), - password=str(secret.get('peertube', 'password')), - client_id=str(secret.get('peertube', 'client_id')), - client_secret=str(secret.get('peertube', 'client_secret')) - ) - except Exception as e: - if hasattr(e, 'message'): - logger.critical("Peertube: " + str(e.message)) - exit(1) - else: - logger.critical("Peertube: " + str(e)) - exit(1) - return oauth - - -def get_default_channel(user_info): - return user_info['videoChannels'][0]['id'] - - -def get_channel_by_name(user_info, options): - for channel in user_info["videoChannels"]: - if channel['displayName'] == options.get('--channel'): - return channel['id'] - - -def convert_peertube_date(date): - date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S') - tz = get_localzone() - tz = pytz.timezone(str(tz)) - return tz.localize(date).isoformat() - - -def create_channel(oauth, url, options): - template = ('Peertube: Channel %s does not exist, creating it.') - logger.info(template % (str(options.get('--channel')))) - channel_name = utils.cleanString(str(options.get('--channel'))) - # Peertube allows 20 chars max for channel name - channel_name = channel_name[:19] - data = '{"name":"' + channel_name + '", \ - "displayName":"' + options.get('--channel') + '", \ - "description":null, \ - "support":null}' - - headers = { - 'Content-Type': "application/json; charset=UTF-8" - } - try: - response = oauth.post(url + "/api/v1/video-channels/", - data=data.encode('utf-8'), - headers=headers) - except Exception as e: - if hasattr(e, 'message'): - logger.error("Peertube: " + str(e.message)) - else: - logger.error("Peertube: " + str(e)) - if response is not None: - if response.status_code == 200: - jresponse = response.json() - jresponse = jresponse['videoChannel'] - return jresponse['id'] - if response.status_code == 409: - logger.critical('Peertube: It seems there is a conflict with an existing channel named ' - + channel_name + '.' - ' Please beware Peertube internal name is compiled from 20 firsts characters of channel name.' - ' Also note that channel name are not case sensitive (no uppercase nor accent)' - ' Please check your channel name and retry.') - exit(1) - else: - logger.critical(('Peertube: Creating channel failed with an unexpected response: ' - '%s') % response) - exit(1) - - -def get_default_playlist(user_info): - return user_info['videoChannels'][0]['id'] - - -def get_playlist_by_name(oauth, url, username, options): - start = 0 - user_playlists = json.loads(oauth.get( - url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content) - total = user_playlists["total"] - data = user_playlists["data"] - # We need to iterate on pagination as peertube returns max 100 playlists (see #41) - while start < total: - for playlist in data: - if playlist['displayName'] == options.get('--playlist'): - return playlist['id'] - start = start + 100 - user_playlists = json.loads(oauth.get( - url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content) - data = user_playlists["data"] - - -def create_playlist(oauth, url, options, channel): - template = ('Peertube: Playlist %s does not exist, creating it.') - logger.info(template % (str(options.get('--playlist')))) - # We use files for form-data Content - # see https://requests.readthedocs.io/en/latest/user/quickstart/#post-a-multipart-encoded-file - # None is used to mute "filename" field - files = {'displayName': (None, str(options.get('--playlist'))), - 'privacy': (None, "1"), - 'description': (None, "null"), - 'videoChannelId': (None, str(channel)), - 'thumbnailfile': (None, "null")} - try: - response = oauth.post(url + "/api/v1/video-playlists/", - files=files) - except Exception as e: - if hasattr(e, 'message'): - logger.error("Peertube: " + str(e.message)) - else: - logger.error("Peertube: " + str(e)) - if response is not None: - if response.status_code == 200: - jresponse = response.json() - jresponse = jresponse['videoPlaylist'] - return jresponse['id'] - else: - logger.critical(('Peertube: Creating the playlist failed with an unexpected response: ' - '%s') % response) - exit(1) - - -def set_playlist(oauth, url, video_id, playlist_id): - logger.info('Peertube: add video to playlist.') - data = '{"videoId":"' + str(video_id) + '"}' - - headers = { - 'Content-Type': "application/json" - } - try: - response = oauth.post(url + "/api/v1/video-playlists/"+str(playlist_id)+"/videos", - data=data, - headers=headers) - except Exception as e: - if hasattr(e, 'message'): - logger.error("Peertube: " + str(e.message)) - else: - logger.error("Peertube: " + str(e)) - if response is not None: - if response.status_code == 200: - logger.info('Peertube: Video is successfully added to the playlist.') - else: - logger.critical(('Peertube: Configuring the playlist failed with an unexpected response: ' - '%s') % response) - exit(1) - - -def upload_video(oauth, secret, options): - - def get_userinfo(): - return json.loads(oauth.get(url+"/api/v1/users/me").content) - - def get_file(path): - mimetypes.init() - return (basename(path), open(abspath(path), 'rb'), - mimetypes.types_map[splitext(path)[1]]) - - path = options.get('--file') - url = str(secret.get('peertube', 'peertube_url')).rstrip('/') - user_info = get_userinfo() - username = str(secret.get('peertube', 'username').lower()) - - # We need to transform fields into tuple to deal with tags as - # MultipartEncoder does not support list refer - # https://github.com/requests/toolbelt/issues/190 and - # https://github.com/requests/toolbelt/issues/205 - fields = [ - ("name", options.get('--name') or splitext(basename(options.get('--file')))[0]), - ("licence", "1"), - ("description", options.get('--description') or "default description"), - ("nsfw", str(int(options.get('--nsfw')) or "0")), - ("videofile", get_file(path)) - ] - - if options.get('--tags'): - tags = options.get('--tags').split(',') - tag_number = 0 - for strtag in tags: - tag_number = tag_number + 1 - # Empty tag crashes Peertube, so skip them - if strtag == "": - continue - # Tag more than 30 chars crashes Peertube, so skip tags - if len(strtag) >= 30: - logger.warning("Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce tag: " + strtag) - logger.warning("Peertube: Meanwhile, this tag will be skipped") - continue - # Peertube supports only 5 tags at the moment - if tag_number > 5: - logger.warning("Peertube: Sorry, Peertube support 5 tags max, additional tag will be skipped") - logger.warning("Peertube: Skipping tag " + strtag) - continue - fields.append(("tags[]", strtag)) - - if options.get('--category'): - fields.append(("category", str(utils.getCategory(options.get('--category'), 'peertube')))) - else: - # if no category, set default to 2 (Films) - fields.append(("category", "2")) - - if options.get('--language'): - fields.append(("language", str(utils.getLanguage(options.get('--language'), "peertube")))) - else: - # if no language, set default to 1 (English) - fields.append(("language", "en")) - - if options.get('--disable-comments'): - fields.append(("commentsEnabled", "0")) - else: - fields.append(("commentsEnabled", "1")) - - privacy = None - if options.get('--privacy'): - privacy = options.get('--privacy').lower() - - # If peertubeAt exists, use instead of publishAt - if options.get('--peertubeAt'): - publishAt = options.get('--peertubeAt') - elif options.get('--publishAt'): - publishAt = options.get('--publishAt') - - if 'publishAt' in locals(): - publishAt = convert_peertube_date(publishAt) - fields.append(("scheduleUpdate[updateAt]", publishAt)) - fields.append(("scheduleUpdate[privacy]", str(PEERTUBE_PRIVACY["public"]))) - fields.append(("privacy", str(PEERTUBE_PRIVACY["private"]))) - else: - fields.append(("privacy", str(PEERTUBE_PRIVACY[privacy or "private"]))) - - # Set originalDate except if the user force no originalDate - if options.get('--originalDate'): - originalDate = convert_peertube_date(options.get('--originalDate')) - fields.append(("originallyPublishedAt", originalDate)) - - if options.get('--thumbnail'): - fields.append(("thumbnailfile", get_file(options.get('--thumbnail')))) - fields.append(("previewfile", get_file(options.get('--thumbnail')))) - - if options.get('--channel'): - channel_id = get_channel_by_name(user_info, options) - if not channel_id and options.get('--channelCreate'): - channel_id = create_channel(oauth, url, options) - elif not channel_id: - logger.warning("Peertube: Channel `" + options.get('--channel') + "` is unknown, using default channel.") - channel_id = get_default_channel(user_info) - else: - channel_id = get_default_channel(user_info) - - fields.append(("channelId", str(channel_id))) - - if options.get('--playlist'): - playlist_id = get_playlist_by_name(oauth, url, username, options) - if not playlist_id and options.get('--playlistCreate'): - playlist_id = create_playlist(oauth, url, options, channel_id) - elif not playlist_id: - logger.critical("Peertube: Playlist `" + options.get('--playlist') + "` does not exist, please set --playlistCreate" - " if you want to create it") - exit(1) - - logger_stdout = None - if options.get('--url-only') or options.get('--batch'): - logger_stdout = logging.getLogger('stdoutlogs') - - encoder = MultipartEncoder(fields) - if options.get('--quiet'): - multipart_data = encoder - else: - progress_callback = create_callback(encoder, options.get('--progress')) - multipart_data = MultipartEncoderMonitor(encoder, progress_callback) - - headers = { - 'Content-Type': multipart_data.content_type - } - response = oauth.post(url + "/api/v1/videos/upload", - data=multipart_data, - headers=headers) - - if response is not None: - if response.status_code == 200: - jresponse = response.json() - jresponse = jresponse['video'] - uuid = jresponse['uuid'] - video_id = str(jresponse['id']) - - logger.info('Peertube: Video was successfully uploaded.') - template = 'Peertube: Watch it at %s/videos/watch/%s.' - logger.info(template % (url, uuid)) - template_stdout = '%s/videos/watch/%s' - if options.get('--url-only'): - logger_stdout.info(template_stdout % (url, uuid)) - elif options.get('--batch'): - logger_stdout.info("Peertube: " + template_stdout % (url, uuid)) - # Upload is successful we may set playlist - if options.get('--playlist'): - set_playlist(oauth, url, video_id, playlist_id) - else: - logger.critical(('Peertube: The upload failed with an unexpected response: ' - '%s') % response) - exit(1) - - -upload_finished = False -def create_callback(encoder, progress_type): - upload_size_MB = encoder.len * (1 / (1024 * 1024)) - - if progress_type is None or "percentage" in progress_type.lower(): - progress_lambda = lambda x: int((x / encoder.len) * 100) # Default to percentage - elif "bigfile" in progress_type.lower(): - progress_lambda = lambda x: x * (1 / (1024 * 1024)) # MB - elif "accurate" in progress_type.lower(): - progress_lambda = lambda x: x * (1 / (1024)) # kB - else: - # Should not happen outside of development when adding partly a progress type - logger.critical("Peertube: Unknown progress type `" + progress_type + "`") - exit(1) - - bar = ProgressBar(expected_size=progress_lambda(encoder.len), label=f"Peertube upload progress ({upload_size_MB:.2f}MB) ", filled_char='=') - - def callback(monitor): - # We want the condition to capture the varible from the parent scope, not a local variable that is created after - global upload_finished - progress = progress_lambda(monitor.bytes_read) - - bar.show(progress) - - if monitor.bytes_read == encoder.len: - if not upload_finished: - # We get two time in the callback with both bytes equals, skip the first - upload_finished = True - else: - # Print a blank line to not (partly) override the progress bar - print() - logger.info("Peertube: Upload finish, Processing…") - - return callback - - -def run(options): - secret = RawConfigParser() - try: - secret.read(PEERTUBE_SECRETS_FILE) - except Exception as e: - logger.critical("Peertube: Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e)) - exit(1) - insecure_transport = secret.get('peertube', 'OAUTHLIB_INSECURE_TRANSPORT') - os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = insecure_transport - oauth = get_authenticated_service(secret) - try: - logger.info('Peertube: Uploading video...') - upload_video(oauth, secret, options) - except Exception as e: - if hasattr(e, 'message'): - logger.error("Peertube: " + str(e.message)) - else: - logger.error("Peertube: " + str(e)) diff --git a/prismedia/upload.py b/prismedia/upload.py index a020eea..fa7f923 100755 --- a/prismedia/upload.py +++ b/prismedia/upload.py @@ -136,112 +136,6 @@ except ImportError: VERSION = "prismedia v0.11.0" -VALID_PRIVACY_STATUSES = ('public', 'private', 'unlisted') -VALID_CATEGORIES = ( - "music", "films", "vehicles", - "sports", "travels", "gaming", "people", - "comedy", "entertainment", "news", - "how to", "education", "activism", "science & technology", - "science", "technology", "animals" -) -VALID_PLATFORM = ('youtube', 'peertube', 'none') -VALID_LANGUAGES = ('arabic', 'english', 'french', - 'german', 'hindi', 'italian', - 'japanese', 'korean', 'mandarin', - 'portuguese', 'punjabi', 'russian', 'spanish') -VALID_PROGRESS = ('percentage', 'bigfile', 'accurate') - - -def validateVideo(path): - supported_types = ['video/mp4'] - detected_type = magic.from_file(path, mime=True) - if detected_type not in supported_types: - print("File", path, "detected type is", detected_type, "which is not one of", supported_types) - - force_file = ['y', 'yes'] - is_forcing = input("Are you sure you selected the correct file? (y/N)") - if is_forcing.lower() not in force_file: - return False - - return path - - -def validateCategory(category): - if category.lower() in VALID_CATEGORIES: - return True - else: - return False - - -def validatePrivacy(privacy): - if privacy.lower() in VALID_PRIVACY_STATUSES: - return True - else: - return False - - -def validatePlatform(platform): - for plfrm in platform.split(','): - if plfrm.lower().replace(" ", "") not in VALID_PLATFORM: - return False - - return True - - -def validateLanguage(language): - if language.lower() in VALID_LANGUAGES: - return True - else: - return False - - -def validatePublishDate(publishDate): - # Check date format and if date is future - try: - now = datetime.datetime.now() - publishAt = datetime.datetime.strptime(publishDate, '%Y-%m-%dT%H:%M:%S') - if now >= publishAt: - return False - except ValueError: - return False - return True - - -def validateOriginalDate(originalDate): - # Check date format and if date is past - try: - now = datetime.datetime.now() - originalDate = datetime.datetime.strptime(originalDate, '%Y-%m-%dT%H:%M:%S') - if now <= originalDate: - return False - except ValueError: - return False - return True - - -def validateThumbnail(thumbnail): - supported_types = ['image/jpg', 'image/jpeg'] - if os.path.exists(thumbnail) and \ - magic.from_file(thumbnail, mime=True) in supported_types: - return thumbnail - else: - return False - - -def validateLogLevel(loglevel): - numeric_level = getattr(logging, loglevel, None) - if not isinstance(numeric_level, int): - return False - return True - - -def validateProgress(progress): - for prgs in progress.split(','): - if prgs.lower().replace(" ", "") not in VALID_PROGRESS: - return False - - return True - def _optionnalOrStrict(key, scope, error): option = key.replace('-', '') @@ -285,6 +179,7 @@ def configureStdoutLogs(): ch_stdout.setFormatter(formatter_stdout) logger_stdout.addHandler(ch_stdout) + def main(): options = docopt(__doc__, version=VERSION) diff --git a/prismedia/utils.py b/prismedia/utils.py index 9cb87c2..345e2ef 100644 --- a/prismedia/utils.py +++ b/prismedia/utils.py @@ -10,94 +10,112 @@ import datetime logger = logging.getLogger('Prismedia') -### CATEGORIES ### -YOUTUBE_CATEGORY = { - "music": 10, - "films": 1, - "vehicles": 2, - "sport": 17, - "travels": 19, - "gaming": 20, - "people": 22, - "comedy": 23, - "entertainment": 24, - "news": 25, - "how to": 26, - "education": 27, - "activism": 29, - "science & technology": 28, - "science": 28, - "technology": 28, - "animals": 15 -} - -PEERTUBE_CATEGORY = { - "music": 1, - "films": 2, - "vehicles": 3, - "sport": 5, - "travels": 6, - "gaming": 7, - "people": 8, - "comedy": 9, - "entertainment": 10, - "news": 11, - "how to": 12, - "education": 13, - "activism": 14, - "science & technology": 15, - "science": 15, - "technology": 15, - "animals": 16 -} - -### LANGUAGES ### -YOUTUBE_LANGUAGE = { - "arabic": 'ar', - "english": 'en', - "french": 'fr', - "german": 'de', - "hindi": 'hi', - "italian": 'it', - "japanese": 'ja', - "korean": 'ko', - "mandarin": 'zh-CN', - "portuguese": 'pt-PT', - "punjabi": 'pa', - "russian": 'ru', - "spanish": 'es' -} - -PEERTUBE_LANGUAGE = { - "arabic": "ar", - "english": "en", - "french": "fr", - "german": "de", - "hindi": "hi", - "italian": "it", - "japanese": "ja", - "korean": "ko", - "mandarin": "zh", - "portuguese": "pt", - "punjabi": "pa", - "russian": "ru", - "spanish": "es" -} -###################### - - -def getCategory(category, platform): - if platform == "youtube": - return YOUTUBE_CATEGORY[category.lower()] + +VALID_PRIVACY_STATUSES = ('public', 'private', 'unlisted') +VALID_CATEGORIES = ( + "music", "films", "vehicles", + "sports", "travels", "gaming", "people", + "comedy", "entertainment", "news", + "how to", "education", "activism", "science & technology", + "science", "technology", "animals" +) +VALID_LANGUAGES = ('arabic', 'english', 'french', + 'german', 'hindi', 'italian', + 'japanese', 'korean', 'mandarin', + 'portuguese', 'punjabi', 'russian', 'spanish') +VALID_PROGRESS = ('percentage', 'bigfile', 'accurate') + + +def validateVideo(path): + supported_types = ['video/mp4'] + detected_type = magic.from_file(path, mime=True) + if detected_type not in supported_types: + print("File", path, "detected type is '" + detected_type + "' which is not one of", supported_types) + + force_file = ['y', 'yes'] + is_forcing = input("Are you sure you selected the correct file? (y/N)") + if is_forcing.lower() not in force_file: + return False + + return path + + +def validateCategory(category): + if category.lower() in VALID_CATEGORIES: + return True else: - return PEERTUBE_CATEGORY[category.lower()] + return False -def getLanguage(language, platform): - if platform == "youtube": - return YOUTUBE_LANGUAGE[language.lower()] +def validatePrivacy(privacy): + if privacy.lower() in VALID_PRIVACY_STATUSES: + return True else: - return PEERTUBE_LANGUAGE[language.lower()] + return False + + +# TODO: remove me? +# def validatePlatform(platform): +# for plfrm in platform.split(','): +# if plfrm.lower().replace(" ", "") not in VALID_PLATFORM: +# return False +# +# return True + + +def validateLanguage(language): + if language.lower() in VALID_LANGUAGES: + return True + else: + return False + + +def validatePublishDate(publishDate): + # Check date format and if date is future + try: + now = datetime.datetime.now() + publishAt = datetime.datetime.strptime(publishDate, '%Y-%m-%dT%H:%M:%S') + if now >= publishAt: + return False + except ValueError: + return False + return True + + +def validateOriginalDate(originalDate): + # Check date format and if date is past + try: + now = datetime.datetime.now() + originalDate = datetime.datetime.strptime(originalDate, '%Y-%m-%dT%H:%M:%S') + if now <= originalDate: + return False + except ValueError: + return False + return True + + +def validateThumbnail(thumbnail): + supported_types = ['image/jpg', 'image/jpeg'] + if os.path.exists(thumbnail) and \ + magic.from_file(thumbnail, mime=True) in supported_types: + return thumbnail + else: + return False + + +def validateLogLevel(loglevel): + numeric_level = getattr(logging, loglevel, None) + if not isinstance(numeric_level, int): + return False + return True + + +def validateProgress(progress): + for prgs in progress.split(','): + if prgs.lower().replace(" ", "") not in VALID_PROGRESS: + return False + + return True def ask_overwrite(question): @@ -225,10 +243,6 @@ def parseNFO(options): return options -def upcaseFirstLetter(s): - return s[0].upper() + s[1:] - - def cleanString(toclean): toclean = unidecode.unidecode(toclean) cleaned = re.sub('[^A-Za-z0-9]+', '', toclean)