prismedia/prismedia/plugins/platforms/peertube.py

#!/usr/bin/env python
# coding: utf-8

import pluginInterfaces as pi
import utils

import mimetypes
import json
import logging
import datetime
import pytz
from os.path import splitext, basename, abspath
from tzlocal import get_localzone

from configparser import RawConfigParser
from requests_oauthlib import OAuth2Session
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
from oauthlib.oauth2 import LegacyApplicationClient
from clint.textui.progress import Bar as ProgressBar
from yapsy.PluginManager import PluginManagerSingleton

logger = logging.getLogger('Prismedia')
upload_finished = False


class Peertube(pi.IPlatformPlugin):
    """
    Plugin to upload to the Peertube platform.
    The connections files should be set as # TODO: EXPLAIN HOW TO SETUP THE SECRET FILES
    - `publish-at-peertube=DATE`: overrides the default `publish-at=DATE` for this platform. # TODO: Maybe we will use a [<plugin_name>] section on the config fire, explain that.
    """
    NAME = "peertube"    # TODO: find if it is possible to get the plugin’s name from inside the plugin
    SECRETS_FILE = "peertube_secret"
    PRIVACY = {
        "public": 1,
        "unlisted": 2,
        "private": 3
    }

    CATEGORY = {
        "music": 1,
        "films": 2,
        "vehicles": 3,
        "sport": 5,
        "travels": 6,
        "gaming": 7,
        "people": 8,
        "comedy": 9,
        "entertainment": 10,
        "news": 11,
        "how to": 12,
        "education": 13,
        "activism": 14,
        "science & technology": 15,
        "science": 15,
        "technology": 15,
        "animals": 16
    }

    LANGUAGE = {
        "arabic": "ar",
        "english": "en",
        "french": "fr",
        "german": "de",
        "hindi": "hi",
        "italian": "it",
        "japanese": "ja",
        "korean": "ko",
        "mandarin": "zh",
        "portuguese": "pt",
        "punjabi": "pa",
        "russian": "ru",
        "spanish": "es"
    }

    def __init__(self):
        self.channelCreate = False
        self.oauth = {}
        self.secret = {}

    def prepare_options(self, video, options):
        pluginManager = PluginManagerSingleton.get()
        # TODO: get the `publish-at-peertube=DATE` option
        # TODO: get the `channel` and `channel-create` options
        pluginManager.registerOptionFromPlugin("Platform", self.NAME, "publish-at", "2034-05-07T19:00:00")
        pluginManager.registerOptionFromPlugin("Platform", self.NAME, "channel", "toto")
        pluginManager.registerOptionFromPlugin("Platform", self.NAME, "channel-create", False)
        video.platform[self.NAME].channel = ""

        self.secret = RawConfigParser()
        self.secret.read(self.SECRETS_FILE)
        self.get_authenticated_service()

        return True

    def get_authenticated_service(self):
        instance_url = str(self.secret.get('peertube', 'peertube_url')).rstrip("/")

        oauth_client = LegacyApplicationClient(
            client_id=str(self.secret.get('peertube', 'client_id'))
        )
        self.oauth = OAuth2Session(client=oauth_client)
        self.oauth.fetch_token(
            token_url=str(instance_url + '/api/v1/users/token'),
            # lower as peertube does not store uppercase for pseudo
            username=str(self.secret.get('peertube', 'username').lower()),
            password=str(self.secret.get('peertube', 'password')),
            client_id=str(self.secret.get('peertube', 'client_id')),
            client_secret=str(self.secret.get('peertube', 'client_secret'))
        )

    def convert_peertube_date(self, date):
        date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')
        tz = get_localzone()
        tz = pytz.timezone(str(tz))
        return tz.localize(date).isoformat()

    def get_default_channel(self, user_info):
        return user_info['videoChannels'][0]['id']

    def get_channel_by_name(self, user_info, video):
        for channel in user_info["videoChannels"]:
            if channel['displayName'] == video.platform[self.NAME].channel:
                return channel['id']

    def create_channel(self, instance_url, video):
        template = ('Peertube: Channel %s does not exist, creating it.')
        logger.info(template % (video.platform[self.NAME].channel))
        channel_name = utils.cleanString(video.platform[self.NAME].channel)
        # Peertube allows 20 chars max for channel name
        channel_name = channel_name[:19]
        data = '{"name":"' + channel_name + '", \
                "displayName":"' + video.platform[self.NAME].channel + '", \
                "description":null, \
                "support":null}'

        headers = {
            'Content-Type': "application/json; charset=UTF-8"
        }
        try:
            response = self.oauth.post(instance_url + "/api/v1/video-channels/",
                                       data=data.encode('utf-8'),
                                       headers=headers)
        except Exception as e:
            logger.error("Peertube: " + utils.get_exception_string(e))

        if response is not None:
            if response.status_code == 200:
                jresponse = response.json()
                jresponse = jresponse['videoChannel']
                return jresponse['id']
            if response.status_code == 409:
                logger.critical('Peertube: It seems there is a conflict with an existing channel named '
                                + channel_name + '.'
                                                 ' Please beware Peertube internal name is compiled from 20 firsts characters of channel name.'
                                                 ' Also note that channel name are not case sensitive (no uppercase nor accent)'
                                                 ' Please check your channel name and retry.')
                exit(1)
            else:
                logger.critical(('Peertube: Creating channel failed with an unexpected response: '
                                 '%s') % response)
                exit(1)

    def get_default_playlist(self, user_info):
        return user_info['videoChannels'][0]['id']

    def get_playlist_by_name(self, instance_url, username, video):
        start = 0
        user_playlists = json.loads(self.oauth.get(
            instance_url + "/api/v1/accounts/" + username + "/video-playlists?start=" + str(
                start) + "&count=100").content)
        total = user_playlists["total"]
        data = user_playlists["data"]
        # We need to iterate on pagination as peertube returns max 100 playlists (see #41)
        while start < total:
            for playlist in data:
                if playlist['displayName'] == video.playlistName:
                    return playlist['id']
            start = start + 100
            user_playlists = json.loads(self.oauth.get(
                instance_url + "/api/v1/accounts/" + username + "/video-playlists?start=" + str(
                    start) + "&count=100").content)
            data = user_playlists["data"]

    def create_playlist(self, instance_url, video, channel):
        template = ('Peertube: Playlist %s does not exist, creating it.')
        logger.info(template % (str(video.playlistName)))
        # We use files for form-data Content
        # see https://requests.readthedocs.io/en/latest/user/quickstart/#post-a-multipart-encoded-file
        # None is used to mute "filename" field
        files = {'displayName': (None, str(video.playlistName)),
                 'privacy': (None, "1"),
                 'description': (None, "null"),
                 'videoChannelId': (None, str(channel)),
                 'thumbnailfile': (None, "null")}

        try:
            response = self.oauth.post(instance_url + "/api/v1/video-playlists/",
                                       files=files)
        except Exception as e:
            logger.error("Peertube: " + utils.get_exception_string(e))

        if response is not None:
            if response.status_code == 200:
                jresponse = response.json()
                jresponse = jresponse['videoPlaylist']
                return jresponse['id']
            else:
                logger.critical(('Peertube: Creating the playlist failed with an unexpected response: '
                                 '%s') % response)
                exit(1)

    def set_playlist(self, instance_url, video_id, playlist_id):
        logger.info('Peertube: add video to playlist.')
        data = '{"videoId":"' + str(video_id) + '"}'

        headers = {
            'Content-Type': "application/json"
        }

        try:
            response = self.oauth.post(instance_url + "/api/v1/video-playlists/" + str(playlist_id) + "/videos",
                                       data=data,
                                       headers=headers)
        except Exception as e:
            logger.error("Peertube: " + utils.get_exception_string(e))

        if response is not None:
            if response.status_code == 200:
                logger.info('Peertube: Video is successfully added to the playlist.')
            else:
                logger.critical(('Peertube: Configuring the playlist failed with an unexpected response: '
                                 '%s') % response)
                exit(1)

    def upload_video(self, video, options):

        def get_userinfo(base_url):
            return json.loads(self.oauth.get(base_url + "/api/v1/users/me").content)

        def get_file(video_path):
            mimetypes.init()
            return (basename(video_path), open(abspath(video_path), 'rb'),
                    mimetypes.types_map[splitext(video_path)[1]])

        path = video.path
        instance_url = str(self.secret.get('peertube', 'peertube_url')).rstrip('/')
        user_info = get_userinfo(instance_url)
        username = str(self.secret.get('peertube', 'username').lower())

        # We need to transform fields into tuple to deal with tags as
        # MultipartEncoder does not support list refer
        # https://github.com/requests/toolbelt/issues/190 and
        # https://github.com/requests/toolbelt/issues/205
        fields = [
            ("name", video.name),
            ("licence", "1"),  # TODO: get licence from video object
            ("description", video.description),
            ("category", str(self.CATEGORY[video.category])),
            ("language", str(self.LANGUAGE[video.language])),
            ("commentsEnabled", "0" if video.disableComments else "1"),
            ("nsfw", "1" if video.nsfw else "0"),
            ("videofile", get_file(path))
        ]

        tag_number = 0
        for strtag in video.tags:
            tag_number = tag_number + 1
            # Empty tag crashes Peertube, so skip them
            if strtag == "":
                continue
            # Tag more than 30 chars crashes Peertube, so skip tags
            if len(strtag) >= 30:
                logger.warning(
                    "Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce tag: " + strtag)
                logger.warning("Peertube: Meanwhile, this tag will be skipped")
                continue
            # Peertube supports only 5 tags at the moment
            if tag_number > 5:
                logger.warning("Peertube: Sorry, Peertube support 5 tags max, additional tag will be skipped")
                logger.warning("Peertube: Skipping tag " + strtag)
                continue
            fields.append(("tags[]", strtag))

        # If peertubeAt exists, use instead of publishAt
        if video.platform[self.NAME].publishAt:
            publishAt = video.platform[self.NAME].publishAt
        elif video.publishAt:
            publishAt = video.publishAt

        if 'publishAt' in locals():
            publishAt = convert_peertube_date(publishAt)
            fields.append(("scheduleUpdate[updateAt]", publishAt))
            fields.append(("scheduleUpdate[privacy]", str(self.PRIVACY["public"])))
            fields.append(("privacy", str(self.PRIVACY["private"])))
        else:
            fields.append(("privacy", str(self.PRIVACY[video.privacy])))

        if video.originalDate:
            originalDate = convert_peertube_date(video.originalDate)
            fields.append(("originallyPublishedAt", originalDate))

        if video.thumbnail:
            fields.append(("thumbnailfile", get_file(video.thumbnail)))
            fields.append(("previewfile", get_file(video.thumbnail)))

        if hasattr(video.platform[self.NAME], "channel"):  # TODO: Should always be present
            channel_id = self.get_channel_by_name(user_info, video)
            if not channel_id and self.channelCreate:
                channel_id = self.create_channel(instance_url, video)
            elif not channel_id:
                logger.warning("Peertube: Channel `" + video.platform[
                    self.NAME].channel + "` is unknown, using default channel.")  # TODO: debate if we should have the same message and behavior than playlist: "does not exist, please set --channelCreate"
                channel_id = self.get_default_channel(user_info)
        else:
            channel_id = self.get_default_channel(user_info)

        fields.append(("channelId", str(channel_id)))

        if video.playlistName:
            playlist_id = get_playlist_by_name(instance_url, username, video)
            if not playlist_id and video.playlistCreate:
                playlist_id = create_playlist(instance_url, video, channel_id)
            elif not playlist_id:
                logger.critical(
                    "Peertube: Playlist `" + video.playlistName + "` does not exist, please set --playlistCreate"
                                                                  " if you want to create it")
                exit(1)

        encoder = MultipartEncoder(fields)
        if options.get('--quiet'):
            multipart_data = encoder
        else:
            progress_callback = self.create_callback(encoder, options.get('--progress'))
            multipart_data = MultipartEncoderMonitor(encoder, progress_callback)

        headers = {
            'Content-Type': multipart_data.content_type
        }
        response = self.oauth.post(instance_url + "/api/v1/videos/upload",
                                   data=multipart_data,
                                   headers=headers)

        if response is not None:
            if response.status_code == 200:
                jresponse = response.json()
                jresponse = jresponse['video']
                uuid = jresponse['uuid']
                video_id = str(jresponse['id'])

                logger.info("Peertube: Video was successfully uploaded.")
                template_url = "%s/videos/watch/%s"
                video.platform[self.NAME].url = template_url % (instance_url, uuid)
                logger.info("Peertube: Watch it at " + video.platform[self.NAME].url + ".")
                # Upload is successful we may set playlist
                if 'playlist_id' in locals():
                    set_playlist(instance_url, video_id, playlist_id)
            else:
                logger.critical(('Peertube: The upload failed with an unexpected response: '
                                 '%s') % response)
                exit(1)


    def create_callback(self, encoder, progress_type):
        upload_size_MB = encoder.len * (1 / (1024 * 1024))

        if progress_type is None or "percentage" in progress_type.lower():
            progress_lambda = lambda x: int((x / encoder.len) * 100) # Default to percentage
        elif "bigfile" in progress_type.lower():
            progress_lambda = lambda x: x * (1 / (1024 * 1024)) # MB
        elif "accurate" in progress_type.lower():
            progress_lambda = lambda x: x * (1 / (1024)) # kB
        else:
            # Should not happen outside of development when adding partly a progress type
            logger.critical("Peertube: Unknown progress type `" + progress_type + "`")
            exit(1)

        bar = ProgressBar(expected_size=progress_lambda(encoder.len), label=f"Peertube upload progress ({upload_size_MB:.2f}MB) ", filled_char='=')

        def callback(monitor):
            # We want the condition to capture the varible from the parent scope, not a local variable that is created after
            global upload_finished
            progress = progress_lambda(monitor.bytes_read)

            bar.show(progress)

            if monitor.bytes_read == encoder.len:
                if not upload_finished:
                    # We get two time in the callback with both bytes equals, skip the first
                    upload_finished = True
                else:
                    # Print a blank line to not (partly) override the progress bar
                    print()
                    logger.info("Peertube: Upload finish, Processing…")

        return callback

    def heartbeat(self):
        """
        If needed for your platform, use a bit of the api so the platform is aware the keys are still in use.
        """
        print("heartbeat for peertube (nothing to do)")
        pass

    # def run(options):
    def upload(self, video, options):
        logger.info('Peertube: Uploading video...')
        self.upload_video(video, options)