WIP basis for integrating a plugin system

go in the prismedia directory, and launch `python core.py`

PLUGINS.md

@@ -0,0 +1,13 @@
+For our plugins we are using [yaspy](http://yapsy.sourceforge.net).
+
+# Types
+For an example of the exact methods required to be recognized as a particular type of plugin, see the concerned interface definition.
+
+## Interface
+Plugins that present an interface (cli, gui, configuration folders, …) for the user to tell Prismedia wich video needs to be uploaded, the infos of the videos, …
+
+## Platform
+Also called uploaders, they are the one doing the actual work of uploading video to a particular platform.
+
+## Consumer
+Thoses do actions once the upload is finished (successful or failed).

prismedia/core.py

@@ -0,0 +1,62 @@
+from yapsy.PluginManager import PluginManager
+import pluginInterfaces as pi
+import logging
+# logging.basicConfig(level=logging.DEBUG)
+
+def loadPlugins(type):
+    # Load the plugins from the plugin directory.
+    # TODO: subdirectories too?
+    manager = PluginManager()
+    manager.setPluginPlaces(["plugins"]) # TODO: Generate the absolute path
+
+    # Define the various categories corresponding to the different
+    # kinds of plugins you have defined
+    manager.setCategoriesFilter({
+        "Interface" : pi.IInterfacePlugin,
+        "Platform" : pi.IPlatformPlugin,
+    })
+
+    manager.collectPlugins()
+
+    # Loop round the plugins and print their names.
+    print("debug")
+    print(manager.getAllPlugins())
+
+    print("all plugins")
+    for plugin in manager.getAllPlugins():
+        plugin.plugin_object.print_name()
+
+    print("Category: Interface")
+    for plugin in manager.getPluginsOfCategory("Interface"):
+        plugin.plugin_object.print_name()
+
+    print("Category: Platform")
+    for plugin in manager.getPluginsOfCategory("Platform"):
+        plugin.plugin_object.print_name()
+
+    # discovered_plugins = {
+    # name: importlib.import_module(name)
+    # for finder, name, ispkg
+    # in pkgutil.iter_modules(["/home/zykino/Documents/0DocPerso/Code/prismedia/plugins"])
+    # if name.startswith("prismedia_" + type + "_")
+    # }
+
+#def test_loadPlugins(arg):
+platforms = loadPlugins("platform")
+print (platforms)
+
+def startInterface():
+    interface = loadPlugins("interface")
+
+    options = interface["default"].run()
+    if options.get('--interface'):
+        if interface[options.get('--interface')]:
+            options = interface[options.get('--interface')].run(options)
+        else:
+            options = interface["cli"].run(options)
+            options = interface["nfo"].run(options)
+
+def uploadToPlatforms(options):
+    platforms = loadPlugins("platform")
+    for platform in options.get('--platform'):
+        platforms[platform].run(options)

prismedia/pluginInterfaces.py

@@ -0,0 +1,60 @@
+from yapsy.IPlugin import IPlugin
+
+###
+# Interface
+###
+# TODO: The interface is not thought out yet
+class IInterfacePlugin(IPlugin):
+	"""
+    Interface for the Interface plugin category.
+	"""
+
+	def getOptions(self, args):
+		"""
+		Returns the options user has set.
+        - `args` the command line arguments passed to Prismedia
+		"""
+		raise NotImplementedError("`getOptions` must be reimplemented by %s" % self)
+
+
+###
+# Platform
+###
+class IPlatformPlugin(IPlugin):
+	"""
+    Interface for the Platform plugin category.
+	"""
+
+	# def dryrun(self, video, options):
+	# 	"""
+    #     Simulate an upload but without really uploading anything.
+	# 	"""
+	# 	raise NotImplementedError("`dryrun` must be reimplemented by %s" % self)
+
+	def hearthbeat(self):
+		"""
+        If needed for your platform, use a bit of the api so the platform is aware the keys are still in use.
+		"""
+		raise NotImplementedError("`hearthbeat` must be reimplemented by %s" % self)
+
+	def upload(self, video, options):
+		"""
+        The upload function
+		"""
+		raise NotImplementedError("`upload` must be reimplemented by %s" % self)
+
+###
+# Consumer
+###
+# TODO: The interface is not thought out yet
+class IConsumerPlugin(IPlugin):
+	"""
+    Interface for the Consumer plugin category.
+	"""
+
+	def finished(self, video):
+		"""
+		What to do once the uploads are done.
+        - `video` is an object containing the video details. The `platforms` key contain a list of the platforms the video has been uploaded to and the status
+		"""
+		raise NotImplementedError("`getOptions` must be reimplemented by %s" % self)

prismedia/plugins/platforms/peertube.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import os
+import mimetypes
+import json
+import logging
+import sys
+import datetime
+import pytz
+import pluginInterfaces as pi
+from os.path import splitext, basename, abspath
+from tzlocal import get_localzone
+
+from configparser import RawConfigParser
+from requests_oauthlib import OAuth2Session
+from oauthlib.oauth2 import LegacyApplicationClient
+from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
+from clint.textui.progress import Bar as ProgressBar
+
+import utils
+logger = logging.getLogger('Prismedia')
+
+PEERTUBE_SECRETS_FILE = 'peertube_secret'
+PEERTUBE_PRIVACY = {
+    "public": 1,
+    "unlisted": 2,
+    "private": 3
+}
+
+CATEGORY = {
+    "music": 1,
+    "films": 2,
+    "vehicles": 3,
+    "sport": 5,
+    "travels": 6,
+    "gaming": 7,
+    "people": 8,
+    "comedy": 9,
+    "entertainment": 10,
+    "news": 11,
+    "how to": 12,
+    "education": 13,
+    "activism": 14,
+    "science & technology": 15,
+    "science": 15,
+    "technology": 15,
+    "animals": 16
+}
+
+LANGUAGE = {
+    "arabic": "ar",
+    "english": "en",
+    "french": "fr",
+    "german": "de",
+    "hindi": "hi",
+    "italian": "it",
+    "japanese": "ja",
+    "korean": "ko",
+    "mandarin": "zh",
+    "portuguese": "pt",
+    "punjabi": "pa",
+    "russian": "ru",
+    "spanish": "es"
+}
+
+class Peertube(pi.IPlatformPlugin):
+    """docstring for Peertube."""
+    def print_name(self):
+        print("This is plugin peertube")
+
+    def get_authenticated_service(secret):
+        peertube_url = str(secret.get('peertube', 'peertube_url')).rstrip("/")
+
+        oauth_client = LegacyApplicationClient(
+            client_id=str(secret.get('peertube', 'client_id'))
+        )
+        try:
+            oauth = OAuth2Session(client=oauth_client)
+            oauth.fetch_token(
+                token_url=str(peertube_url + '/api/v1/users/token'),
+                # lower as peertube does not store uppercase for pseudo
+                username=str(secret.get('peertube', 'username').lower()),
+                password=str(secret.get('peertube', 'password')),
+                client_id=str(secret.get('peertube', 'client_id')),
+                client_secret=str(secret.get('peertube', 'client_secret'))
+            )
+        except Exception as e:
+            if hasattr(e, 'message'):
+                logger.critical("Peertube: " + str(e.message))
+                exit(1)
+            else:
+                logger.critical("Peertube: " + str(e))
+                exit(1)
+        return oauth
+
+
+    def get_default_channel(user_info):
+        return user_info['videoChannels'][0]['id']
+
+
+    def get_channel_by_name(user_info, options):
+        for channel in user_info["videoChannels"]:
+            if channel['displayName'] == options.get('--channel'):
+                return channel['id']
+
+
+    def convert_peertube_date(date):
+        date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')
+        tz = get_localzone()
+        tz = pytz.timezone(str(tz))
+        return tz.localize(date).isoformat()
+
+
+    def create_channel(oauth, url, options):
+        template = ('Peertube: Channel %s does not exist, creating it.')
+        logger.info(template % (str(options.get('--channel'))))
+        channel_name = utils.cleanString(str(options.get('--channel')))
+        # Peertube allows 20 chars max for channel name
+        channel_name = channel_name[:19]
+        data = '{"name":"' + channel_name + '", \
+                "displayName":"' + options.get('--channel') + '", \
+                "description":null, \
+                "support":null}'
+
+        headers = {
+            'Content-Type': "application/json; charset=UTF-8"
+        }
+        try:
+            response = oauth.post(url + "/api/v1/video-channels/",
+                                  data=data.encode('utf-8'),
+                                  headers=headers)
+        except Exception as e:
+            if hasattr(e, 'message'):
+                logger.error("Peertube: " + str(e.message))
+            else:
+                logger.error("Peertube: " + str(e))
+        if response is not None:
+            if response.status_code == 200:
+                jresponse = response.json()
+                jresponse = jresponse['videoChannel']
+                return jresponse['id']
+            if response.status_code == 409:
+                logger.critical('Peertube: It seems there is a conflict with an existing channel named '
+                              + channel_name + '.'
+                              ' Please beware Peertube internal name is compiled from 20 firsts characters of channel name.'
+                              ' Also note that channel name are not case sensitive (no uppercase nor accent)'
+                              ' Please check your channel name and retry.')
+                exit(1)
+            else:
+                logger.critical(('Peertube: Creating channel failed with an unexpected response: '
+                               '%s') % response)
+                exit(1)
+
+
+    def get_default_playlist(user_info):
+        return user_info['videoChannels'][0]['id']
+
+
+    def get_playlist_by_name(oauth, url, username, options):
+        start = 0
+        user_playlists = json.loads(oauth.get(
+            url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content)
+        total = user_playlists["total"]
+        data = user_playlists["data"]
+        # We need to iterate on pagination as peertube returns max 100 playlists (see #41)
+        while start < total:
+            for playlist in data:
+                if playlist['displayName'] == options.get('--playlist'):
+                    return playlist['id']
+            start = start + 100
+            user_playlists = json.loads(oauth.get(
+                url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content)
+            data = user_playlists["data"]
+
+
+    def create_playlist(oauth, url, options, channel):
+        template = ('Peertube: Playlist %s does not exist, creating it.')
+        logger.info(template % (str(options.get('--playlist'))))
+        # We use files for form-data Content
+        # see https://requests.readthedocs.io/en/latest/user/quickstart/#post-a-multipart-encoded-file
+        # None is used to mute "filename" field
+        files = {'displayName': (None, str(options.get('--playlist'))),
+                 'privacy': (None, "1"),
+                 'description': (None, "null"),
+                 'videoChannelId': (None, str(channel)),
+                 'thumbnailfile': (None, "null")}
+        try:
+            response = oauth.post(url + "/api/v1/video-playlists/",
+                           files=files)
+        except Exception as e:
+            if hasattr(e, 'message'):
+                logger.error("Peertube: " + str(e.message))
+            else:
+                logger.error("Peertube: " + str(e))
+        if response is not None:
+            if response.status_code == 200:
+                jresponse = response.json()
+                jresponse = jresponse['videoPlaylist']
+                return jresponse['id']
+            else:
+                logger.critical(('Peertube: Creating the playlist failed with an unexpected response: '
+                               '%s') % response)
+                exit(1)
+
+
+    def set_playlist(oauth, url, video_id, playlist_id):
+        logger.info('Peertube: add video to playlist.')
+        data = '{"videoId":"' + str(video_id) + '"}'
+
+        headers = {
+            'Content-Type': "application/json"
+        }
+        try:
+            response = oauth.post(url + "/api/v1/video-playlists/"+str(playlist_id)+"/videos",
+                                  data=data,
+                                  headers=headers)
+        except Exception as e:
+            if hasattr(e, 'message'):
+                logger.error("Peertube: " + str(e.message))
+            else:
+                logger.error("Peertube: " + str(e))
+        if response is not None:
+            if response.status_code == 200:
+                logger.info('Peertube: Video is successfully added to the playlist.')
+            else:
+                logger.critical(('Peertube: Configuring the playlist failed with an unexpected response: '
+                               '%s') % response)
+                exit(1)
+
+
+    def upload_video(oauth, secret, options):
+
+        def get_userinfo():
+            return json.loads(oauth.get(url+"/api/v1/users/me").content)
+
+        def get_file(path):
+            mimetypes.init()
+            return (basename(path), open(abspath(path), 'rb'),
+                    mimetypes.types_map[splitext(path)[1]])
+
+        path = options.get('--file')
+        url = str(secret.get('peertube', 'peertube_url')).rstrip('/')
+        user_info = get_userinfo()
+        username = str(secret.get('peertube', 'username').lower())
+
+        # We need to transform fields into tuple to deal with tags as
+        # MultipartEncoder does not support list refer
+        # https://github.com/requests/toolbelt/issues/190 and
+        # https://github.com/requests/toolbelt/issues/205
+        fields = [
+            ("name", options.get('--name') or splitext(basename(options.get('--file')))[0]),
+            ("licence", "1"),
+            ("description", options.get('--description')  or "default description"),
+            ("nsfw", str(int(options.get('--nsfw')) or "0")),
+            ("videofile", get_file(path))
+        ]
+
+        if options.get('--tags'):
+            tags = options.get('--tags').split(',')
+            tag_number = 0
+            for strtag in tags:
+                tag_number = tag_number + 1
+                # Empty tag crashes Peertube, so skip them
+                if strtag == "":
+                    continue
+                # Tag more than 30 chars crashes Peertube, so skip tags
+                if len(strtag) >= 30:
+                    logger.warning("Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce tag: " + strtag)
+                    logger.warning("Peertube: Meanwhile, this tag will be skipped")
+                    continue
+                # Peertube supports only 5 tags at the moment
+                if tag_number > 5:
+                    logger.warning("Peertube: Sorry, Peertube support 5 tags max, additional tag will be skipped")
+                    logger.warning("Peertube: Skipping tag " + strtag)
+                    continue
+                fields.append(("tags[]", strtag))
+
+        if options.get('--category'):
+            fields.append(("category", str(CATEGORY[options.get('--category').lower()])))
+        else:
+            # if no category, set default to 2 (Films)
+            fields.append(("category", "2"))
+
+        if options.get('--language'):
+            fields.append(("language", str(LANGUAGE[options.get('--language').lower()])))
+        else:
+            # if no language, set default to 1 (English)
+            fields.append(("language", "en"))
+
+        if options.get('--disable-comments'):
+            fields.append(("commentsEnabled", "0"))
+        else:
+            fields.append(("commentsEnabled", "1"))
+
+        privacy = None
+        if options.get('--privacy'):
+            privacy = options.get('--privacy').lower()
+
+        # If peertubeAt exists, use instead of publishAt
+        if options.get('--peertubeAt'):
+            publishAt = options.get('--peertubeAt')
+        elif options.get('--publishAt'):
+            publishAt = options.get('--publishAt')
+
+        if 'publishAt' in locals():
+            publishAt = convert_peertube_date(publishAt)
+            fields.append(("scheduleUpdate[updateAt]", publishAt))
+            fields.append(("scheduleUpdate[privacy]", str(PEERTUBE_PRIVACY["public"])))
+            fields.append(("privacy", str(PEERTUBE_PRIVACY["private"])))
+        else:
+            fields.append(("privacy", str(PEERTUBE_PRIVACY[privacy or "private"])))
+
+        # Set originalDate except if the user force no originalDate
+        if options.get('--originalDate'):
+            originalDate = convert_peertube_date(options.get('--originalDate'))
+            fields.append(("originallyPublishedAt", originalDate))
+
+        if options.get('--thumbnail'):
+            fields.append(("thumbnailfile", get_file(options.get('--thumbnail'))))
+            fields.append(("previewfile", get_file(options.get('--thumbnail'))))
+
+        if options.get('--channel'):
+            channel_id = get_channel_by_name(user_info, options)
+            if not channel_id and options.get('--channelCreate'):
+                channel_id = create_channel(oauth, url, options)
+            elif not channel_id:
+                logger.warning("Peertube: Channel `" + options.get('--channel') + "` is unknown, using default channel.")
+                channel_id = get_default_channel(user_info)
+        else:
+            channel_id = get_default_channel(user_info)
+
+        fields.append(("channelId", str(channel_id)))
+
+        if options.get('--playlist'):
+            playlist_id = get_playlist_by_name(oauth, url, username, options)
+            if not playlist_id and options.get('--playlistCreate'):
+                playlist_id = create_playlist(oauth, url, options, channel_id)
+            elif not playlist_id:
+                logger.critical("Peertube: Playlist `" + options.get('--playlist') + "` does not exist, please set --playlistCreate"
+                                " if you want to create it")
+                exit(1)
+
+        logger_stdout = None
+        if options.get('--url-only') or options.get('--batch'):
+            logger_stdout = logging.getLogger('stdoutlogs')
+
+        encoder = MultipartEncoder(fields)
+        if options.get('--quiet'):
+            multipart_data = encoder
+        else:
+            progress_callback = create_callback(encoder, options.get('--progress'))
+            multipart_data = MultipartEncoderMonitor(encoder, progress_callback)
+
+        headers = {
+            'Content-Type': multipart_data.content_type
+        }
+        response = oauth.post(url + "/api/v1/videos/upload",
+                              data=multipart_data,
+                              headers=headers)
+
+        if response is not None:
+            if response.status_code == 200:
+                jresponse = response.json()
+                jresponse = jresponse['video']
+                uuid = jresponse['uuid']
+                video_id = str(jresponse['id'])
+
+                logger.info('Peertube: Video was successfully uploaded.')
+                template = 'Peertube: Watch it at %s/videos/watch/%s.'
+                logger.info(template % (url, uuid))
+                template_stdout = '%s/videos/watch/%s'
+                if options.get('--url-only'):
+                    logger_stdout.info(template_stdout % (url, uuid))
+                elif options.get('--batch'):
+                    logger_stdout.info("Peertube: " + template_stdout % (url, uuid))
+                # Upload is successful we may set playlist
+                if options.get('--playlist'):
+                    set_playlist(oauth, url, video_id, playlist_id)
+            else:
+                logger.critical(('Peertube: The upload failed with an unexpected response: '
+                               '%s') % response)
+                exit(1)
+
+
+    upload_finished = False
+    def create_callback(encoder, progress_type):
+        upload_size_MB = encoder.len * (1 / (1024 * 1024))
+
+        if progress_type is None or "percentage" in progress_type.lower():
+            progress_lambda = lambda x: int((x / encoder.len) * 100) # Default to percentage
+        elif "bigfile" in progress_type.lower():
+            progress_lambda = lambda x: x * (1 / (1024 * 1024)) # MB
+        elif "accurate" in progress_type.lower():
+            progress_lambda = lambda x: x * (1 / (1024)) # kB
+        else:
+            # Should not happen outside of development when adding partly a progress type
+            logger.critical("Peertube: Unknown progress type `" + progress_type + "`")
+            exit(1)
+
+        bar = ProgressBar(expected_size=progress_lambda(encoder.len), label=f"Peertube upload progress ({upload_size_MB:.2f}MB) ", filled_char='=')
+
+        def callback(monitor):
+            # We want the condition to capture the varible from the parent scope, not a local variable that is created after
+            global upload_finished
+            progress = progress_lambda(monitor.bytes_read)
+
+            bar.show(progress)
+
+            if monitor.bytes_read == encoder.len:
+                if not upload_finished:
+                    # We get two time in the callback with both bytes equals, skip the first
+                    upload_finished = True
+                else:
+                    # Print a blank line to not (partly) override the progress bar
+                    print()
+                    logger.info("Peertube: Upload finish, Processing…")
+
+        return callback
+
+
+    def hearthbeat(self):
+        """
+        If needed for your platform, use a bit of the api so the platform is aware the keys are still in use.
+        """
+        pass
+
+
+    def upload(self, video, options):
+    # def run(options):
+        secret = RawConfigParser()
+        try:
+            secret.read(PEERTUBE_SECRETS_FILE)
+        except Exception as e:
+            logger.critical("Peertube: Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e))
+            exit(1)
+        insecure_transport = secret.get('peertube', 'OAUTHLIB_INSECURE_TRANSPORT')
+        os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = insecure_transport
+        oauth = get_authenticated_service(secret)
+        try:
+            logger.info('Peertube: Uploading video...')
+            upload_video(oauth, secret, options)
+        except Exception as e:
+            if hasattr(e, 'message'):
+                logger.error("Peertube: " + str(e.message))
+            else:
+                logger.error("Peertube: " + str(e))

prismedia/plugins/platforms/peertube.yapsy-plugin

@@ -0,0 +1,9 @@
+[Core]
+Name = Peertube
+Module = peertube
+
+[Documentation]
+Author = Le Cygne Noir
+Version = 0.1
+Website = https://git.lecygnenoir.info/LecygneNoir/prismedia
+Description = Upload to the peertube platform

prismedia/yt_upload.py → prismedia/plugins/platforms/youtube.py

@@ -54,6 +54,41 @@ SCOPES = ['https://www.googleapis.com/auth/youtube.upload', 'https://www.googlea
 API_SERVICE_NAME = 'youtube'
 API_VERSION = 'v3'
 
+CATEGORY = {
+    "music": 10,
+    "films": 1,
+    "vehicles": 2,
+    "sport": 17,
+    "travels": 19,
+    "gaming": 20,
+    "people": 22,
+    "comedy": 23,
+    "entertainment": 24,
+    "news": 25,
+    "how to": 26,
+    "education": 27,
+    "activism": 29,
+    "science & technology": 28,
+    "science": 28,
+    "technology": 28,
+    "animals": 15
+}
+
+LANGUAGE = {
+    "arabic": 'ar',
+    "english": 'en',
+    "french": 'fr',
+    "german": 'de',
+    "hindi": 'hi',
+    "italian": 'it',
+    "japanese": 'ja',
+    "korean": 'ko',
+    "mandarin": 'zh-CN',
+    "portuguese": 'pt-PT',
+    "punjabi": 'pa',
+    "russian": 'ru',
+    "spanish": 'es'
+}
 
 # Authorize the request and store authorization credentials.
 def get_authenticated_service():
@@ -107,11 +142,11 @@ def initialize_upload(youtube, options):
 
     category = None
     if options.get('--category'):
-        category = utils.getCategory(options.get('--category'), 'youtube')
+        category = CATEGORY[options.get('--category').lower()]
 
     language = None
     if options.get('--language'):
-        language = utils.getLanguage(options.get('--language'), "youtube")
+        language = LANGUAGE[options.get('--language').lower()]
 
     license = None
     if options.get('--cca'):

prismedia/pt_upload.py

@@ -1,398 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-import os
-import mimetypes
-import json
-import logging
-import sys
-import datetime
-import pytz
-from os.path import splitext, basename, abspath
-from tzlocal import get_localzone
-
-from configparser import RawConfigParser
-from requests_oauthlib import OAuth2Session
-from oauthlib.oauth2 import LegacyApplicationClient
-from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
-from clint.textui.progress import Bar as ProgressBar
-
-from . import utils
-logger = logging.getLogger('Prismedia')
-
-PEERTUBE_SECRETS_FILE = 'peertube_secret'
-PEERTUBE_PRIVACY = {
-    "public": 1,
-    "unlisted": 2,
-    "private": 3
-}
-
-
-def get_authenticated_service(secret):
-    peertube_url = str(secret.get('peertube', 'peertube_url')).rstrip("/")
-
-    oauth_client = LegacyApplicationClient(
-        client_id=str(secret.get('peertube', 'client_id'))
-    )
-    try:
-        oauth = OAuth2Session(client=oauth_client)
-        oauth.fetch_token(
-            token_url=str(peertube_url + '/api/v1/users/token'),
-            # lower as peertube does not store uppercase for pseudo
-            username=str(secret.get('peertube', 'username').lower()),
-            password=str(secret.get('peertube', 'password')),
-            client_id=str(secret.get('peertube', 'client_id')),
-            client_secret=str(secret.get('peertube', 'client_secret'))
-        )
-    except Exception as e:
-        if hasattr(e, 'message'):
-            logger.critical("Peertube: " + str(e.message))
-            exit(1)
-        else:
-            logger.critical("Peertube: " + str(e))
-            exit(1)
-    return oauth
-
-
-def get_default_channel(user_info):
-    return user_info['videoChannels'][0]['id']
-
-
-def get_channel_by_name(user_info, options):
-    for channel in user_info["videoChannels"]:
-        if channel['displayName'] == options.get('--channel'):
-            return channel['id']
-
-
-def convert_peertube_date(date):
-    date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')
-    tz = get_localzone()
-    tz = pytz.timezone(str(tz))
-    return tz.localize(date).isoformat()
-
-
-def create_channel(oauth, url, options):
-    template = ('Peertube: Channel %s does not exist, creating it.')
-    logger.info(template % (str(options.get('--channel'))))
-    channel_name = utils.cleanString(str(options.get('--channel')))
-    # Peertube allows 20 chars max for channel name
-    channel_name = channel_name[:19]
-    data = '{"name":"' + channel_name + '", \
-            "displayName":"' + options.get('--channel') + '", \
-            "description":null, \
-            "support":null}'
-
-    headers = {
-        'Content-Type': "application/json; charset=UTF-8"
-    }
-    try:
-        response = oauth.post(url + "/api/v1/video-channels/",
-                              data=data.encode('utf-8'),
-                              headers=headers)
-    except Exception as e:
-        if hasattr(e, 'message'):
-            logger.error("Peertube: " + str(e.message))
-        else:
-            logger.error("Peertube: " + str(e))
-    if response is not None:
-        if response.status_code == 200:
-            jresponse = response.json()
-            jresponse = jresponse['videoChannel']
-            return jresponse['id']
-        if response.status_code == 409:
-            logger.critical('Peertube: It seems there is a conflict with an existing channel named '
-                          + channel_name + '.'
-                          ' Please beware Peertube internal name is compiled from 20 firsts characters of channel name.'
-                          ' Also note that channel name are not case sensitive (no uppercase nor accent)'
-                          ' Please check your channel name and retry.')
-            exit(1)
-        else:
-            logger.critical(('Peertube: Creating channel failed with an unexpected response: '
-                           '%s') % response)
-            exit(1)
-
-
-def get_default_playlist(user_info):
-    return user_info['videoChannels'][0]['id']
-
-
-def get_playlist_by_name(oauth, url, username, options):
-    start = 0
-    user_playlists = json.loads(oauth.get(
-        url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content)
-    total = user_playlists["total"]
-    data = user_playlists["data"]
-    # We need to iterate on pagination as peertube returns max 100 playlists (see #41)
-    while start < total:
-        for playlist in data:
-            if playlist['displayName'] == options.get('--playlist'):
-                return playlist['id']
-        start = start + 100
-        user_playlists = json.loads(oauth.get(
-            url+"/api/v1/accounts/"+username+"/video-playlists?start="+str(start)+"&count=100").content)
-        data = user_playlists["data"]
-
-
-def create_playlist(oauth, url, options, channel):
-    template = ('Peertube: Playlist %s does not exist, creating it.')
-    logger.info(template % (str(options.get('--playlist'))))
-    # We use files for form-data Content
-    # see https://requests.readthedocs.io/en/latest/user/quickstart/#post-a-multipart-encoded-file
-    # None is used to mute "filename" field
-    files = {'displayName': (None, str(options.get('--playlist'))),
-             'privacy': (None, "1"),
-             'description': (None, "null"),
-             'videoChannelId': (None, str(channel)),
-             'thumbnailfile': (None, "null")}
-    try:
-        response = oauth.post(url + "/api/v1/video-playlists/",
-                       files=files)
-    except Exception as e:
-        if hasattr(e, 'message'):
-            logger.error("Peertube: " + str(e.message))
-        else:
-            logger.error("Peertube: " + str(e))
-    if response is not None:
-        if response.status_code == 200:
-            jresponse = response.json()
-            jresponse = jresponse['videoPlaylist']
-            return jresponse['id']
-        else:
-            logger.critical(('Peertube: Creating the playlist failed with an unexpected response: '
-                           '%s') % response)
-            exit(1)
-
-
-def set_playlist(oauth, url, video_id, playlist_id):
-    logger.info('Peertube: add video to playlist.')
-    data = '{"videoId":"' + str(video_id) + '"}'
-
-    headers = {
-        'Content-Type': "application/json"
-    }
-    try:
-        response = oauth.post(url + "/api/v1/video-playlists/"+str(playlist_id)+"/videos",
-                              data=data,
-                              headers=headers)
-    except Exception as e:
-        if hasattr(e, 'message'):
-            logger.error("Peertube: " + str(e.message))
-        else:
-            logger.error("Peertube: " + str(e))
-    if response is not None:
-        if response.status_code == 200:
-            logger.info('Peertube: Video is successfully added to the playlist.')
-        else:
-            logger.critical(('Peertube: Configuring the playlist failed with an unexpected response: '
-                           '%s') % response)
-            exit(1)
-
-
-def upload_video(oauth, secret, options):
-
-    def get_userinfo():
-        return json.loads(oauth.get(url+"/api/v1/users/me").content)
-
-    def get_file(path):
-        mimetypes.init()
-        return (basename(path), open(abspath(path), 'rb'),
-                mimetypes.types_map[splitext(path)[1]])
-
-    path = options.get('--file')
-    url = str(secret.get('peertube', 'peertube_url')).rstrip('/')
-    user_info = get_userinfo()
-    username = str(secret.get('peertube', 'username').lower())
-
-    # We need to transform fields into tuple to deal with tags as
-    # MultipartEncoder does not support list refer
-    # https://github.com/requests/toolbelt/issues/190 and
-    # https://github.com/requests/toolbelt/issues/205
-    fields = [
-        ("name", options.get('--name') or splitext(basename(options.get('--file')))[0]),
-        ("licence", "1"),
-        ("description", options.get('--description')  or "default description"),
-        ("nsfw", str(int(options.get('--nsfw')) or "0")),
-        ("videofile", get_file(path))
-    ]
-
-    if options.get('--tags'):
-        tags = options.get('--tags').split(',')
-        tag_number = 0
-        for strtag in tags:
-            tag_number = tag_number + 1
-            # Empty tag crashes Peertube, so skip them
-            if strtag == "":
-                continue
-            # Tag more than 30 chars crashes Peertube, so skip tags
-            if len(strtag) >= 30:
-                logger.warning("Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce tag: " + strtag)
-                logger.warning("Peertube: Meanwhile, this tag will be skipped")
-                continue
-            # Peertube supports only 5 tags at the moment
-            if tag_number > 5:
-                logger.warning("Peertube: Sorry, Peertube support 5 tags max, additional tag will be skipped")
-                logger.warning("Peertube: Skipping tag " + strtag)
-                continue
-            fields.append(("tags[]", strtag))
-
-    if options.get('--category'):
-        fields.append(("category", str(utils.getCategory(options.get('--category'), 'peertube'))))
-    else:
-        # if no category, set default to 2 (Films)
-        fields.append(("category", "2"))
-
-    if options.get('--language'):
-        fields.append(("language", str(utils.getLanguage(options.get('--language'), "peertube"))))
-    else:
-        # if no language, set default to 1 (English)
-        fields.append(("language", "en"))
-
-    if options.get('--disable-comments'):
-        fields.append(("commentsEnabled", "0"))
-    else:
-        fields.append(("commentsEnabled", "1"))
-
-    privacy = None
-    if options.get('--privacy'):
-        privacy = options.get('--privacy').lower()
-
-    # If peertubeAt exists, use instead of publishAt
-    if options.get('--peertubeAt'):
-        publishAt = options.get('--peertubeAt')
-    elif options.get('--publishAt'):
-        publishAt = options.get('--publishAt')
-
-    if 'publishAt' in locals():
-        publishAt = convert_peertube_date(publishAt)
-        fields.append(("scheduleUpdate[updateAt]", publishAt))
-        fields.append(("scheduleUpdate[privacy]", str(PEERTUBE_PRIVACY["public"])))
-        fields.append(("privacy", str(PEERTUBE_PRIVACY["private"])))
-    else:
-        fields.append(("privacy", str(PEERTUBE_PRIVACY[privacy or "private"])))
-
-    # Set originalDate except if the user force no originalDate
-    if options.get('--originalDate'):
-        originalDate = convert_peertube_date(options.get('--originalDate'))
-        fields.append(("originallyPublishedAt", originalDate))
-
-    if options.get('--thumbnail'):
-        fields.append(("thumbnailfile", get_file(options.get('--thumbnail'))))
-        fields.append(("previewfile", get_file(options.get('--thumbnail'))))
-
-    if options.get('--channel'):
-        channel_id = get_channel_by_name(user_info, options)
-        if not channel_id and options.get('--channelCreate'):
-            channel_id = create_channel(oauth, url, options)
-        elif not channel_id:
-            logger.warning("Peertube: Channel `" + options.get('--channel') + "` is unknown, using default channel.")
-            channel_id = get_default_channel(user_info)
-    else:
-        channel_id = get_default_channel(user_info)
-
-    fields.append(("channelId", str(channel_id)))
-
-    if options.get('--playlist'):
-        playlist_id = get_playlist_by_name(oauth, url, username, options)
-        if not playlist_id and options.get('--playlistCreate'):
-            playlist_id = create_playlist(oauth, url, options, channel_id)
-        elif not playlist_id:
-            logger.critical("Peertube: Playlist `" + options.get('--playlist') + "` does not exist, please set --playlistCreate"
-                            " if you want to create it")
-            exit(1)
-
-    logger_stdout = None
-    if options.get('--url-only') or options.get('--batch'):
-        logger_stdout = logging.getLogger('stdoutlogs')
-
-    encoder = MultipartEncoder(fields)
-    if options.get('--quiet'):
-        multipart_data = encoder
-    else:
-        progress_callback = create_callback(encoder, options.get('--progress'))
-        multipart_data = MultipartEncoderMonitor(encoder, progress_callback)
-
-    headers = {
-        'Content-Type': multipart_data.content_type
-    }
-    response = oauth.post(url + "/api/v1/videos/upload",
-                          data=multipart_data,
-                          headers=headers)
-
-    if response is not None:
-        if response.status_code == 200:
-            jresponse = response.json()
-            jresponse = jresponse['video']
-            uuid = jresponse['uuid']
-            video_id = str(jresponse['id'])
-
-            logger.info('Peertube: Video was successfully uploaded.')
-            template = 'Peertube: Watch it at %s/videos/watch/%s.'
-            logger.info(template % (url, uuid))
-            template_stdout = '%s/videos/watch/%s'
-            if options.get('--url-only'):
-                logger_stdout.info(template_stdout % (url, uuid))
-            elif options.get('--batch'):
-                logger_stdout.info("Peertube: " + template_stdout % (url, uuid))
-            # Upload is successful we may set playlist
-            if options.get('--playlist'):
-                set_playlist(oauth, url, video_id, playlist_id)
-        else:
-            logger.critical(('Peertube: The upload failed with an unexpected response: '
-                           '%s') % response)
-            exit(1)
-
-
-upload_finished = False
-def create_callback(encoder, progress_type):
-    upload_size_MB = encoder.len * (1 / (1024 * 1024))
-
-    if progress_type is None or "percentage" in progress_type.lower():
-        progress_lambda = lambda x: int((x / encoder.len) * 100) # Default to percentage
-    elif "bigfile" in progress_type.lower():
-        progress_lambda = lambda x: x * (1 / (1024 * 1024)) # MB
-    elif "accurate" in progress_type.lower():
-        progress_lambda = lambda x: x * (1 / (1024)) # kB
-    else:
-        # Should not happen outside of development when adding partly a progress type
-        logger.critical("Peertube: Unknown progress type `" + progress_type + "`")
-        exit(1)
-
-    bar = ProgressBar(expected_size=progress_lambda(encoder.len), label=f"Peertube upload progress ({upload_size_MB:.2f}MB) ", filled_char='=')
-
-    def callback(monitor):
-        # We want the condition to capture the varible from the parent scope, not a local variable that is created after
-        global upload_finished
-        progress = progress_lambda(monitor.bytes_read)
-
-        bar.show(progress)
-
-        if monitor.bytes_read == encoder.len:
-            if not upload_finished:
-                # We get two time in the callback with both bytes equals, skip the first
-                upload_finished = True
-            else:
-                # Print a blank line to not (partly) override the progress bar
-                print()
-                logger.info("Peertube: Upload finish, Processing…")
-
-    return callback
-
-
-def run(options):
-    secret = RawConfigParser()
-    try:
-        secret.read(PEERTUBE_SECRETS_FILE)
-    except Exception as e:
-        logger.critical("Peertube: Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e))
-        exit(1)
-    insecure_transport = secret.get('peertube', 'OAUTHLIB_INSECURE_TRANSPORT')
-    os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = insecure_transport
-    oauth = get_authenticated_service(secret)
-    try:
-        logger.info('Peertube: Uploading video...')
-        upload_video(oauth, secret, options)
-    except Exception as e:
-        if hasattr(e, 'message'):
-            logger.error("Peertube: " + str(e.message))
-        else:
-            logger.error("Peertube: " + str(e))

prismedia/upload.py

@@ -136,112 +136,6 @@ except ImportError:
 
 VERSION = "prismedia v0.11.0"
 
-VALID_PRIVACY_STATUSES = ('public', 'private', 'unlisted')
-VALID_CATEGORIES = (
-    "music", "films", "vehicles",
-    "sports", "travels", "gaming", "people",
-    "comedy", "entertainment", "news",
-    "how to", "education", "activism", "science & technology",
-    "science", "technology", "animals"
-)
-VALID_PLATFORM = ('youtube', 'peertube', 'none')
-VALID_LANGUAGES = ('arabic', 'english', 'french',
-                   'german', 'hindi', 'italian',
-                   'japanese', 'korean', 'mandarin',
-                   'portuguese', 'punjabi', 'russian', 'spanish')
-VALID_PROGRESS = ('percentage', 'bigfile', 'accurate')
-
-
-def validateVideo(path):
-    supported_types = ['video/mp4']
-    detected_type = magic.from_file(path, mime=True)
-    if detected_type not in supported_types:
-        print("File", path, "detected type is", detected_type, "which is not one of", supported_types)
-
-        force_file = ['y', 'yes']
-        is_forcing = input("Are you sure you selected the correct file? (y/N)")
-        if is_forcing.lower() not in force_file:
-            return False
-
-    return path
-
-
-def validateCategory(category):
-    if category.lower() in VALID_CATEGORIES:
-        return True
-    else:
-        return False
-
-
-def validatePrivacy(privacy):
-    if privacy.lower() in VALID_PRIVACY_STATUSES:
-        return True
-    else:
-        return False
-
-
-def validatePlatform(platform):
-    for plfrm in platform.split(','):
-        if plfrm.lower().replace(" ", "") not in VALID_PLATFORM:
-            return False
-
-    return True
-
-
-def validateLanguage(language):
-    if language.lower() in VALID_LANGUAGES:
-        return True
-    else:
-        return False
-
-
-def validatePublishDate(publishDate):
-    # Check date format and if date is future
-    try:
-        now = datetime.datetime.now()
-        publishAt = datetime.datetime.strptime(publishDate, '%Y-%m-%dT%H:%M:%S')
-        if now >= publishAt:
-            return False
-    except ValueError:
-        return False
-    return True
-
-
-def validateOriginalDate(originalDate):
-    # Check date format and if date is past
-    try:
-        now = datetime.datetime.now()
-        originalDate = datetime.datetime.strptime(originalDate, '%Y-%m-%dT%H:%M:%S')
-        if now <= originalDate:
-            return False
-    except ValueError:
-        return False
-    return True
-
-
-def validateThumbnail(thumbnail):
-    supported_types = ['image/jpg', 'image/jpeg']
-    if os.path.exists(thumbnail) and \
-            magic.from_file(thumbnail, mime=True) in supported_types:
-        return thumbnail
-    else:
-        return False
-
-
-def validateLogLevel(loglevel):
-    numeric_level = getattr(logging, loglevel, None)
-    if not isinstance(numeric_level, int):
-        return False
-    return True
-
-
-def validateProgress(progress):
-    for prgs in progress.split(','):
-        if prgs.lower().replace(" ", "") not in VALID_PROGRESS:
-            return False
-
-    return True
-
 
 def _optionnalOrStrict(key, scope, error):
     option = key.replace('-', '')
@@ -285,6 +179,7 @@ def configureStdoutLogs():
     ch_stdout.setFormatter(formatter_stdout)
     logger_stdout.addHandler(ch_stdout)
 
+
 def main():
     options = docopt(__doc__, version=VERSION)
 

prismedia/utils.py

@@ -10,94 +10,112 @@ import datetime
 
 logger = logging.getLogger('Prismedia')
 
-### CATEGORIES ###
-YOUTUBE_CATEGORY = {
-    "music": 10,
-    "films": 1,
-    "vehicles": 2,
-    "sport": 17,
-    "travels": 19,
-    "gaming": 20,
-    "people": 22,
-    "comedy": 23,
-    "entertainment": 24,
-    "news": 25,
-    "how to": 26,
-    "education": 27,
-    "activism": 29,
-    "science & technology": 28,
-    "science": 28,
-    "technology": 28,
-    "animals": 15
-}
-
-PEERTUBE_CATEGORY = {
-    "music": 1,
-    "films": 2,
-    "vehicles": 3,
-    "sport": 5,
-    "travels": 6,
-    "gaming": 7,
-    "people": 8,
-    "comedy": 9,
-    "entertainment": 10,
-    "news": 11,
-    "how to": 12,
-    "education": 13,
-    "activism": 14,
-    "science & technology": 15,
-    "science": 15,
-    "technology": 15,
-    "animals": 16
-}
-
-### LANGUAGES ###
-YOUTUBE_LANGUAGE = {
-    "arabic": 'ar',
-    "english": 'en',
-    "french": 'fr',
-    "german": 'de',
-    "hindi": 'hi',
-    "italian": 'it',
-    "japanese": 'ja',
-    "korean": 'ko',
-    "mandarin": 'zh-CN',
-    "portuguese": 'pt-PT',
-    "punjabi": 'pa',
-    "russian": 'ru',
-    "spanish": 'es'
-}
-
-PEERTUBE_LANGUAGE = {
-    "arabic": "ar",
-    "english": "en",
-    "french": "fr",
-    "german": "de",
-    "hindi": "hi",
-    "italian": "it",
-    "japanese": "ja",
-    "korean": "ko",
-    "mandarin": "zh",
-    "portuguese": "pt",
-    "punjabi": "pa",
-    "russian": "ru",
-    "spanish": "es"
-}
-######################
-
-
-def getCategory(category, platform):
-    if platform == "youtube":
-        return YOUTUBE_CATEGORY[category.lower()]
+
+VALID_PRIVACY_STATUSES = ('public', 'private', 'unlisted')
+VALID_CATEGORIES = (
+    "music", "films", "vehicles",
+    "sports", "travels", "gaming", "people",
+    "comedy", "entertainment", "news",
+    "how to", "education", "activism", "science & technology",
+    "science", "technology", "animals"
+)
+VALID_LANGUAGES = ('arabic', 'english', 'french',
+                   'german', 'hindi', 'italian',
+                   'japanese', 'korean', 'mandarin',
+                   'portuguese', 'punjabi', 'russian', 'spanish')
+VALID_PROGRESS = ('percentage', 'bigfile', 'accurate')
+
+
+def validateVideo(path):
+    supported_types = ['video/mp4']
+    detected_type = magic.from_file(path, mime=True)
+    if detected_type not in supported_types:
+        print("File", path, "detected type is '" + detected_type + "' which is not one of", supported_types)
+
+        force_file = ['y', 'yes']
+        is_forcing = input("Are you sure you selected the correct file? (y/N)")
+        if is_forcing.lower() not in force_file:
+            return False
+
+    return path
+
+
+def validateCategory(category):
+    if category.lower() in VALID_CATEGORIES:
+        return True
     else:
-        return PEERTUBE_CATEGORY[category.lower()]
+        return False
 
 
-def getLanguage(language, platform):
-    if platform == "youtube":
-        return YOUTUBE_LANGUAGE[language.lower()]
+def validatePrivacy(privacy):
+    if privacy.lower() in VALID_PRIVACY_STATUSES:
+        return True
     else:
-        return PEERTUBE_LANGUAGE[language.lower()]
+        return False
+
+
+# TODO: remove me?
+# def validatePlatform(platform):
+#     for plfrm in platform.split(','):
+#         if plfrm.lower().replace(" ", "") not in VALID_PLATFORM:
+#             return False
+#
+#     return True
+
+
+def validateLanguage(language):
+    if language.lower() in VALID_LANGUAGES:
+        return True
+    else:
+        return False
+
+
+def validatePublishDate(publishDate):
+    # Check date format and if date is future
+    try:
+        now = datetime.datetime.now()
+        publishAt = datetime.datetime.strptime(publishDate, '%Y-%m-%dT%H:%M:%S')
+        if now >= publishAt:
+            return False
+    except ValueError:
+        return False
+    return True
+
+
+def validateOriginalDate(originalDate):
+    # Check date format and if date is past
+    try:
+        now = datetime.datetime.now()
+        originalDate = datetime.datetime.strptime(originalDate, '%Y-%m-%dT%H:%M:%S')
+        if now <= originalDate:
+            return False
+    except ValueError:
+        return False
+    return True
+
+
+def validateThumbnail(thumbnail):
+    supported_types = ['image/jpg', 'image/jpeg']
+    if os.path.exists(thumbnail) and \
+            magic.from_file(thumbnail, mime=True) in supported_types:
+        return thumbnail
+    else:
+        return False
+
+
+def validateLogLevel(loglevel):
+    numeric_level = getattr(logging, loglevel, None)
+    if not isinstance(numeric_level, int):
+        return False
+    return True
+
+
+def validateProgress(progress):
+    for prgs in progress.split(','):
+        if prgs.lower().replace(" ", "") not in VALID_PROGRESS:
+            return False
+
+    return True
 
 
 def ask_overwrite(question):
@@ -225,10 +243,6 @@ def parseNFO(options):
     return options
 
 
-def upcaseFirstLetter(s):
-    return s[0].upper() + s[1:]
-
-
 def cleanString(toclean):
     toclean = unidecode.unidecode(toclean)
     cleaned = re.sub('[^A-Za-z0-9]+', '', toclean)