From c822a10d0e2705d2ec75af707c13464a6b6820c6 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 28 Jul 2018 12:50:53 +0200 Subject: [PATCH 01/38] Add utilities to manage thumbnail --- lib/utils.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/utils.py b/lib/utils.py index bd7b9d8..f1958e1 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -98,6 +98,32 @@ def getLanguage(language, platform): return PEERTUBE_LANGUAGE[language.lower()] +def remove_empty_kwargs(**kwargs): + good_kwargs = {} + if kwargs is not None: + for key, value in kwargs.iteritems(): + if value: + good_kwargs[key] = value + return good_kwargs + +def searchThumbnail(options): + video_directory = dirname(options.get('--file')) + "/" + # First, check for thumbnail based on videoname + if options.get('--name'): + if isfile(video_directory + options.get('--name') + ".jpg"): + options['--thumbnail'] = video_directory + options.get('--name') + ".jpg" + elif isfile(video_directory + options.get('--name') + ".jpeg"): + options['--thumbnail'] = video_directory + options.get('--name') + ".jpeg" + # Then, if we still not have thumbnail, check for thumbnail based on videofile name + if not options.get('--thumbnail'): + video_file = splitext(basename(options.get('--file')))[0] + if isfile(video_directory + video_file + ".jpg"): + options['--thumbnail'] = video_directory + video_file + ".jpg" + elif isfile(video_directory + video_file + ".jpeg"): + options['--thumbnail'] = video_directory + video_file + ".jpeg" + return options + + # return the nfo as a RawConfigParser object def loadNFO(options): video_directory = dirname(options.get('--file')) + "/" @@ -117,7 +143,6 @@ def loadNFO(options): else: if options.get('--name'): nfo_file = video_directory + options.get('--name') + ".txt" - print nfo_file if isfile(nfo_file): try: logging.info("Using " + nfo_file + " as NFO, loading...") From b442f15b1785c5ed144ba355491724566aeef077 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 28 Jul 2018 12:51:08 +0200 Subject: [PATCH 02/38] Add thumbnail support for peertube --- lib/pt_upload.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index e30d1f4..2a782b1 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -47,12 +47,11 @@ def upload_video(oauth, secret, options): user_info = json.loads(oauth.get(url + "/api/v1/users/me").content) return str(user_info["id"]) - def get_videofile(path): + def get_file(path): mimetypes.init() return (basename(path), open(abspath(path), 'rb'), mimetypes.types_map[splitext(path)[1]]) - path = options.get('--file') url = secret.get('peertube', 'peertube_url') # We need to transform fields into tuple to deal with tags as @@ -65,7 +64,7 @@ def upload_video(oauth, secret, options): ("description", options.get('--description') or "default description"), ("nsfw", str(int(options.get('--nsfw')) or "0")), ("channelId", get_userinfo()), - ("videofile", get_videofile(path)) + ("videofile", get_file(options.get('--file'))) ] if options.get('--tags'): @@ -76,7 +75,7 @@ def upload_video(oauth, secret, options): continue # Tag more than 30 chars crashes Peertube, so exit and check tags if len(strtag) >= 30: - logging.warning("Sorry, Peertube does not support tag with more than 30 characters, please reduce your tag size") + logging.warning("Peertube: Sorry, Peertube does not support tag with more than 30 characters, please reduce your tag size") exit(1) # If Mastodon compatibility is enabled, clean tags from special characters if options.get('--mt'): @@ -105,6 +104,9 @@ def upload_video(oauth, secret, options): else: fields.append(("commentsEnabled", "1")) + if options.get('--thumbnail'): + fields.append(("thumbnailfile", get_file(options.get('--thumbnail')))) + multipart_data = MultipartEncoder(fields) headers = { @@ -120,13 +122,13 @@ def upload_video(oauth, secret, options): jresponse = jresponse['video'] uuid = jresponse['uuid'] idvideo = str(jresponse['id']) - template = ('Peertube : Video was successfully uploaded.\n' - 'Watch it at %s/videos/watch/%s.') + logging.info('Peertube : Video was successfully uploaded.') + template = 'Peertube: Watch it at %s/videos/watch/%s.' logging.info(template % (url, uuid)) if options.get('--publishAt'): utils.publishAt(str(options.get('--publishAt')), oauth, url, idvideo, secret) else: - logging.error(('Peertube : The upload failed with an unexpected response: ' + logging.error(('Peertube: The upload failed with an unexpected response: ' '%s') % response) exit(1) @@ -136,16 +138,16 @@ def run(options): try: secret.read(PEERTUBE_SECRETS_FILE) except Exception as e: - logging.error("Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e)) + logging.error("Peertube: Error loading " + str(PEERTUBE_SECRETS_FILE) + ": " + str(e)) exit(1) insecure_transport = secret.get('peertube', 'OAUTHLIB_INSECURE_TRANSPORT') os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = insecure_transport oauth = get_authenticated_service(secret) try: - logging.info('Peertube : Uploading file...') + logging.info('Peertube: Uploading video...') upload_video(oauth, secret, options) except Exception as e: if hasattr(e, 'message'): - logging.error("Error: " + str(e.message)) + logging.error("Peertube: Error: " + str(e.message)) else: - logging.error("Error: " + str(e)) + logging.error("Peertube: Error: " + str(e)) From 48030691783458aaafa65f28291ca60f54ccc74e Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 28 Jul 2018 12:51:21 +0200 Subject: [PATCH 03/38] Add thumbnail support for youtube --- lib/yt_upload.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/yt_upload.py b/lib/yt_upload.py index 28e9c6f..df58e04 100644 --- a/lib/yt_upload.py +++ b/lib/yt_upload.py @@ -20,6 +20,7 @@ from googleapiclient.errors import HttpError from googleapiclient.http import MediaFileUpload from google_auth_oauthlib.flow import InstalledAppFlow + import utils logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) @@ -126,25 +127,44 @@ def initialize_upload(youtube, options): body=body, media_body=MediaFileUpload(path, chunksize=-1, resumable=True) ) - resumable_upload(insert_request) + video_id = resumable_upload(insert_request, 'video', 'insert') + + # If we get a video_id, upload is successful and we are able to set thumbnail + if video_id and options.get('--thumbnail'): + set_thumbnail(youtube, options.get('--thumbnail'), videoId=video_id) + + +def set_thumbnail(youtube, media_file, **kwargs): + kwargs = utils.remove_empty_kwargs(**kwargs) # See full sample for function + request = youtube.thumbnails().set( + media_body=MediaFileUpload(media_file, chunksize=-1, + resumable=True), + **kwargs + ) + + # See full sample for function + return resumable_upload(request, 'thumbnail', 'set') # This method implements an exponential backoff strategy to resume a # failed upload. -def resumable_upload(request): +def resumable_upload(request, resource, method): response = None error = None retry = 0 while response is None: try: - logging.info('Youtube : Uploading file...') + template = 'Youtube: Uploading %s...' + logging.info(template % resource) status, response = request.next_chunk() if response is not None: - if 'id' in response: - template = ('Youtube : Video was successfully ' - 'uploaded.\n' - 'Watch it at https://youtu.be/%s (post-encoding could take some time)') + if method == 'insert' and 'id' in response: + logging.info('Youtube : Video was successfully uploaded.') + template = 'Youtube: Watch it at https://youtu.be/%s (post-encoding could take some time)' logging.info(template % response['id']) + return response['id'] + elif method != 'insert' or "id" not in response: + logging.info('Youtube: Thumbnail was successfully set.') else: template = ('Youtube : The upload failed with an ' 'unexpected response: %s') From 5896522df5d56964edae0b4e4f57f292fed9f264 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 28 Jul 2018 12:51:44 +0200 Subject: [PATCH 04/38] Update help and base script to manage thumbnail --- nfo_example.txt | 1 + prismedia_upload.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/nfo_example.txt b/nfo_example.txt index 065601d..dc538df 100644 --- a/nfo_example.txt +++ b/nfo_example.txt @@ -14,6 +14,7 @@ category = Films cca = True privacy = private disable-comments = True +thumbnail = /path/to/your/thumbnail.jpg # Set the absolute path to your thumbnail nsfw = True platform = youtube, peertube language = French diff --git a/prismedia_upload.py b/prismedia_upload.py index 7261cbf..508e5ef 100755 --- a/prismedia_upload.py +++ b/prismedia_upload.py @@ -34,6 +34,9 @@ Options: DATE should be on the form YYYY-MM-DDThh:mm:ss eg: 2018-03-12T19:00:00 DATE should be in the future For Peertube, requires the "atd" and "curl utilities installed on the system + --thumbnail=STRING Path to a file to use as a thumbnail for the video. + Supported types are jpg and jpeg. + By default, prismedia search for an image based on video name followed by .jpg or .jpeg -h --help Show this help. --version Show version. @@ -151,6 +154,12 @@ def validatePublish(publish): return False return True +def validateThumbnail(thumbnail): + supported_types = ['image/jpg', 'image/jpeg'] + if magic.from_file(thumbnail, mime=True) in supported_types: + return thumbnail + else: + return False if __name__ == '__main__': @@ -199,12 +208,18 @@ if __name__ == '__main__': Optional('--cca'): bool, Optional('--disable-comments'): bool, Optional('--nsfw'): bool, + Optional('--thumbnail'): Or(None, And( + str, validateThumbnail, error='thumbnail is not supported, please use jpg/jpeg'), + ), '--help': bool, '--version': bool }) options = utils.parseNFO(options) + if not options.get('--thumbnail'): + options = utils.searchThumbnail(options) + try: options = schema.validate(options) except SchemaError as e: From b784a5b5155f2587d5590af8eae1de9f7f6449d3 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 28 Jul 2018 12:52:01 +0200 Subject: [PATCH 05/38] Update README with new features and help for thumbnail --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fd7ec79..2197a95 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,12 @@ Specify description and tags: ./prismedia_upload.py --file="yourvideo.mp4" -d "My supa description" -t "tag1,tag2,foo" ``` +Provide a thumbnail: + +``` +./prismedia_upload.py --file="yourvideo.mp4" -d "Video with thumbnail" --thumbnail="/path/to/your/thumbnail.jpg" +``` + Use a NFO file to specify your video options: @@ -111,7 +117,10 @@ Options: --publishAt=DATE Publish the video at the given DATE using local server timezone. DATE should be on the form YYYY-MM-DDThh:mm:ss eg: 2018-03-12T19:00:00 DATE should be in the future - For Peertube, requires the "atd", "curl" and "jq" utilities installed on the system + For Peertube, requires the "atd" and "curl utilities installed on the system + --thumbnail=STRING Path to a file to use as a thumbnail for the video. + Supported types are jpg and jpeg. + By default, prismedia search for an image based on video name followed by .jpg or .jpeg -h --help Show this help. --version Show version. @@ -145,8 +154,8 @@ Languages: - [x] enabling/disabling comment (Peertube only as Youtube API does not support it) - [x] nsfw (Peertube only as Youtube API does not support it) - [x] set default language - - [ ] thumbnail/preview (YT workflow: upload video, upload thumbnail, add thumbnail to video) - - [ ] multiple lines description (see [issue 4](https://git.lecygnenoir.info/LecygneNoir/prismedia/issues/4)) + - [x] thumbnail/preview + - [x] multiple lines description (see [issue 4](https://git.lecygnenoir.info/LecygneNoir/prismedia/issues/4)) - [ ] add videos to playlist (YT & PT workflow: upload video, find playlist id, add video to playlist) - [x] Use a config file (NFO) file to retrieve videos arguments - [x] Allow to choose peertube or youtube upload (to resume failed upload for example) From 6dd929a7c846d64d3490c933002855b891e85188 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Wed, 1 Aug 2018 21:25:48 +0200 Subject: [PATCH 06/38] Fix old variable path inside peertube upload --- lib/pt_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index 2a782b1..650aa78 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -59,7 +59,7 @@ def upload_video(oauth, secret, options): # https://github.com/requests/toolbelt/issues/190 and # https://github.com/requests/toolbelt/issues/205 fields = [ - ("name", options.get('--name') or splitext(basename(path))[0]), + ("name", options.get('--name') or splitext(basename(options.get('--file')))[0]), ("licence", "1"), ("description", options.get('--description') or "default description"), ("nsfw", str(int(options.get('--nsfw')) or "0")), From 9b0da1ea6534577a3383d14c2380b6a6280aa82c Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Wed, 1 Aug 2018 21:36:15 +0200 Subject: [PATCH 07/38] Add Changelog info for thumbnail feature --- CHANGELOG.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5586b2e..0a95b0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,18 @@ -# Prismedia v0.5 +# Changelog -## Features +## v 0.? + +### Features + - Add the possibility to upload thumbnail + +## v0.5 + +### Features - plan your Peertube videos! Stable release - Support for Peertube beta4 - More examples in NFO - Better support for multilines descriptions -## Fix +### Fixes - Display datetime for output - plan video only if upload is successful \ No newline at end of file From 6a07bbbf4bb93952647b6043f4ba6afde7209155 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 13:50:39 +0200 Subject: [PATCH 08/38] Update peeror repo with new url --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd7ec79..8d864ba 100644 --- a/README.md +++ b/README.md @@ -160,4 +160,4 @@ Languages: If your server uses peertube before 1.0.0-beta4, use the version inside tag 1.0.0-beta3! ## Sources -inspired by [peeror](https://git.drycat.fr/rigelk/Peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) \ No newline at end of file +inspired by [peeror](https://git.rigelk.eu/rigelk/peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) \ No newline at end of file From 68959cc1a8887d661618c1e0f78dba55eb438593 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 14:07:02 +0200 Subject: [PATCH 09/38] Add support for playlist on Peertube --- lib/pt_upload.py | 52 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index e30d1f4..8195336 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -41,11 +41,46 @@ def get_authenticated_service(secret): return oauth +def get_playlist_by_name(user_info, options): + + for playlist in user_info["videoChannels"]: + if playlist['displayName'] == options.get('--playlist'): + return playlist['id'] + + +def create_playlist(oauth, url, options): + template = ('Peertube : Playlist %s does not exist, creating it.') + logging.info(template % (str(options.get('--playlist')))) + data = '{"displayName":"' + str(options.get('--playlist')) +'", \ + "description":null}' + + headers = { + 'Content-Type': "application/json" + } + try: + response = oauth.post(url + "/api/v1/video-channels/", + data=data, + headers=headers) + except Exception as e: + if hasattr(e, 'message'): + logging.error("Error: " + str(e.message)) + else: + logging.error("Error: " + str(e)) + if response is not None: + if response.status_code == 200: + jresponse = response.json() + jresponse = jresponse['videoChannel'] + return jresponse['id'] + else: + logging.error(('Peertube : The upload failed with an unexpected response: ' + '%s') % response) + exit(1) + + def upload_video(oauth, secret, options): def get_userinfo(): - user_info = json.loads(oauth.get(url + "/api/v1/users/me").content) - return str(user_info["id"]) + return json.loads(oauth.get(url+"/api/v1/users/me").content) def get_videofile(path): mimetypes.init() @@ -54,6 +89,7 @@ def upload_video(oauth, secret, options): path = options.get('--file') url = secret.get('peertube', 'peertube_url') + user_info = get_userinfo() # We need to transform fields into tuple to deal with tags as # MultipartEncoder does not support list refer @@ -64,7 +100,6 @@ def upload_video(oauth, secret, options): ("licence", "1"), ("description", options.get('--description') or "default description"), ("nsfw", str(int(options.get('--nsfw')) or "0")), - ("channelId", get_userinfo()), ("videofile", get_videofile(path)) ] @@ -105,12 +140,21 @@ def upload_video(oauth, secret, options): else: fields.append(("commentsEnabled", "1")) + if options.get('--playlist'): + playlist_id = get_playlist_by_name(user_info, options) + if not playlist_id and options.get('--playlistCreate'): + playlist_id = create_playlist(oauth, url, options) + else: + playlist_id = user_info['id'] + else: + playlist_id = user_info['id'] + fields.append(("channelId", str(playlist_id))) + multipart_data = MultipartEncoder(fields) headers = { 'Content-Type': multipart_data.content_type } - response = oauth.post(url + "/api/v1/videos/upload", data=multipart_data, headers=headers) From bd2631699a42fa1300826894430a589a479be86f Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 14:07:37 +0200 Subject: [PATCH 10/38] Add option to manage playlist for videos --- prismedia_upload.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/prismedia_upload.py b/prismedia_upload.py index 7261cbf..297da76 100755 --- a/prismedia_upload.py +++ b/prismedia_upload.py @@ -34,6 +34,10 @@ Options: DATE should be on the form YYYY-MM-DDThh:mm:ss eg: 2018-03-12T19:00:00 DATE should be in the future For Peertube, requires the "atd" and "curl utilities installed on the system + --playlist=STRING Set the playlist to use for the video. Also known as Channel for Peertube. + If the playlist is not found, spawn an error except if --playlist-create is set. + --playlistCreate Create the playlist if not exists. (default do not create) + Only relevant if --playlist is set. -h --help Show this help. --version Show version. @@ -86,7 +90,7 @@ except ImportError: 'see https://github.com/ahupp/python-magic\n') exit(1) -VERSION = "prismedia v0.4" +VERSION = "prismedia v0.5" VALID_PRIVACY_STATUSES = ('public', 'private', 'unlisted') VALID_CATEGORIES = ( @@ -199,6 +203,8 @@ if __name__ == '__main__': Optional('--cca'): bool, Optional('--disable-comments'): bool, Optional('--nsfw'): bool, + Optional('--playlist'): Or(None, str), + Optional('--playlistCreate'): bool, '--help': bool, '--version': bool }) From a160b2e409b88fa5d3ca80b101d6f21bac56f862 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sun, 5 Aug 2018 11:28:22 +0200 Subject: [PATCH 11/38] Use thumbnail as preview for peertube, fix #7 --- lib/pt_upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index 650aa78..4b49f27 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -106,6 +106,7 @@ def upload_video(oauth, secret, options): if options.get('--thumbnail'): fields.append(("thumbnailfile", get_file(options.get('--thumbnail')))) + fields.append(("previewfile", get_file(options.get('--thumbnail')))) multipart_data = MultipartEncoder(fields) From 2e600bc4557a98b85572e70befa94310bdd4c21a Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sun, 5 Aug 2018 11:57:59 +0200 Subject: [PATCH 12/38] Fix bug with trailing / in peertube url. fix #6 --- lib/pt_upload.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index 4b49f27..fa3225d 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -23,21 +23,28 @@ PEERTUBE_PRIVACY = { def get_authenticated_service(secret): - peertube_url = str(secret.get('peertube', 'peertube_url')) + peertube_url = str(secret.get('peertube', 'peertube_url')).rstrip("/") oauth_client = LegacyApplicationClient( client_id=str(secret.get('peertube', 'client_id')) ) - oauth = OAuth2Session(client=oauth_client) - oauth.fetch_token( - token_url=peertube_url + '/api/v1/users/token', - # lower as peertube does not store uppecase for pseudo - username=str(secret.get('peertube', 'username').lower()), - password=str(secret.get('peertube', 'password')), - client_id=str(secret.get('peertube', 'client_id')), - client_secret=str(secret.get('peertube', 'client_secret')) - ) - + try: + oauth = OAuth2Session(client=oauth_client) + oauth.fetch_token( + token_url=str(peertube_url + '/api/v1/users/token'), + # lower as peertube does not store uppercase for pseudo + username=str(secret.get('peertube', 'username').lower()), + password=str(secret.get('peertube', 'password')), + client_id=str(secret.get('peertube', 'client_id')), + client_secret=str(secret.get('peertube', 'client_secret')) + ) + except Exception as e: + if hasattr(e, 'message'): + logging.error("Peertube: Error: " + str(e.message)) + exit(1) + else: + logging.error("Peertube: Error: " + str(e)) + exit(1) return oauth @@ -52,7 +59,7 @@ def upload_video(oauth, secret, options): return (basename(path), open(abspath(path), 'rb'), mimetypes.types_map[splitext(path)[1]]) - url = secret.get('peertube', 'peertube_url') + url = str(secret.get('peertube', 'peertube_url')).rstrip('/') # We need to transform fields into tuple to deal with tags as # MultipartEncoder does not support list refer From f365eb1089302fc26cc5bad7c543466c653ead42 Mon Sep 17 00:00:00 2001 From: Zykino Date: Sun, 9 Sep 2018 17:21:59 +0200 Subject: [PATCH 13/38] fix README with the version of python used and adding a missing dependency --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2197a95..713bf4c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Prismedia -A scripting way to upload videos to peertube and youtube +A scripting way to upload videos to peertube and youtube written in python2 ## Dependencies Search in your package manager, otherwise use ``pip install --upgrade`` @@ -11,6 +11,7 @@ Search in your package manager, otherwise use ``pip install --upgrade`` - docopt - schema - python-magic + - python-magic-bin - requests-toolbelt - tzlocal @@ -60,12 +61,12 @@ Simply upload a video: ``` ./prismedia_upload.py --file="yourvideo.mp4" -``` +``` Specify description and tags: -``` +``` ./prismedia_upload.py --file="yourvideo.mp4" -d "My supa description" -t "tag1,tag2,foo" ``` @@ -168,5 +169,5 @@ Languages: If your server uses peertube before 1.0.0-beta4, use the version inside tag 1.0.0-beta3! -## Sources -inspired by [peeror](https://git.drycat.fr/rigelk/Peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) \ No newline at end of file +## Sources +inspired by [peeror](https://git.drycat.fr/rigelk/Peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) From 844173f326d0a9849de5e0dc1269b1e8316e33f7 Mon Sep 17 00:00:00 2001 From: Zykino Date: Sun, 9 Sep 2018 17:29:09 +0200 Subject: [PATCH 14/38] fix OAuth URL for peertube --- peertube_secret.sample | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/peertube_secret.sample b/peertube_secret.sample index 5051bcb..f2fa723 100644 --- a/peertube_secret.sample +++ b/peertube_secret.sample @@ -1,5 +1,5 @@ # This information is obtained upon registration/once logged in a new PeerTube -# on url+'/oauth-clients/local' +# on url+'/api/v1/oauth-clients/local' # ex: http://domain.example/api/v1/oauth-clients/local # Warn, no quote " inside this file [peertube] @@ -8,4 +8,4 @@ client_secret = your_client_secret username = LecygneNoir password = your_secure_pwd peertube_url = https://domain.example -OAUTHLIB_INSECURE_TRANSPORT = '0' #Default use https \ No newline at end of file +OAUTHLIB_INSECURE_TRANSPORT = '0' #Default use https From 34103c49f2c9943b091b2364ec9d7c88b2fb3576 Mon Sep 17 00:00:00 2001 From: Zykino Date: Sun, 9 Sep 2018 21:19:34 +0200 Subject: [PATCH 15/38] print the peertube response.json when in error --- lib/pt_upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index fa3225d..a5961a2 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -138,6 +138,7 @@ def upload_video(oauth, secret, options): else: logging.error(('Peertube: The upload failed with an unexpected response: ' '%s') % response) + print(response.json()) exit(1) From 3b76221f29e24c2bd759e6b3e1ae9a7295a64053 Mon Sep 17 00:00:00 2001 From: Zykino Date: Tue, 11 Sep 2018 22:08:30 +0200 Subject: [PATCH 16/38] update shebang to use python2 instead of the distrb prefered --- lib/pt_upload.py | 2 +- lib/yt_upload.py | 2 +- prismedia_upload.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index a5961a2..a765de9 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python2 # coding: utf-8 import os diff --git a/lib/yt_upload.py b/lib/yt_upload.py index df58e04..b28d495 100644 --- a/lib/yt_upload.py +++ b/lib/yt_upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python2 # coding: utf-8 # From Youtube samples : https://raw.githubusercontent.com/youtube/api-samples/master/python/upload_video.py # noqa diff --git a/prismedia_upload.py b/prismedia_upload.py index 27d8b8d..4634440 100755 --- a/prismedia_upload.py +++ b/prismedia_upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python2 # coding: utf-8 """ From c5aeacb936e1e7d2113b62c9ccfe40ffc9c34860 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 13:50:39 +0200 Subject: [PATCH 17/38] Update peeror repo with new url --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2197a95..f612c76 100644 --- a/README.md +++ b/README.md @@ -169,4 +169,4 @@ Languages: If your server uses peertube before 1.0.0-beta4, use the version inside tag 1.0.0-beta3! ## Sources -inspired by [peeror](https://git.drycat.fr/rigelk/Peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) \ No newline at end of file +inspired by [peeror](https://git.rigelk.eu/rigelk/peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) \ No newline at end of file From bd8aa9c4850cbc5597dc397117ce059371ee087e Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 14:07:02 +0200 Subject: [PATCH 18/38] Add support for playlist on Peertube --- lib/pt_upload.py | 57 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index fa3225d..afbb8ff 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -48,18 +48,55 @@ def get_authenticated_service(secret): return oauth +def get_playlist_by_name(user_info, options): + + for playlist in user_info["videoChannels"]: + if playlist['displayName'] == options.get('--playlist'): + return playlist['id'] + + +def create_playlist(oauth, url, options): + template = ('Peertube : Playlist %s does not exist, creating it.') + logging.info(template % (str(options.get('--playlist')))) + data = '{"displayName":"' + str(options.get('--playlist')) +'", \ + "description":null}' + + headers = { + 'Content-Type': "application/json" + } + try: + response = oauth.post(url + "/api/v1/video-channels/", + data=data, + headers=headers) + except Exception as e: + if hasattr(e, 'message'): + logging.error("Error: " + str(e.message)) + else: + logging.error("Error: " + str(e)) + if response is not None: + if response.status_code == 200: + jresponse = response.json() + jresponse = jresponse['videoChannel'] + return jresponse['id'] + else: + logging.error(('Peertube : The upload failed with an unexpected response: ' + '%s') % response) + exit(1) + + def upload_video(oauth, secret, options): def get_userinfo(): - user_info = json.loads(oauth.get(url + "/api/v1/users/me").content) - return str(user_info["id"]) + return json.loads(oauth.get(url+"/api/v1/users/me").content) def get_file(path): mimetypes.init() return (basename(path), open(abspath(path), 'rb'), mimetypes.types_map[splitext(path)[1]]) - url = str(secret.get('peertube', 'peertube_url')).rstrip('/') + path = options.get('--file') + url = secret.get('peertube', 'peertube_url') + user_info = get_userinfo() # We need to transform fields into tuple to deal with tags as # MultipartEncoder does not support list refer @@ -70,8 +107,7 @@ def upload_video(oauth, secret, options): ("licence", "1"), ("description", options.get('--description') or "default description"), ("nsfw", str(int(options.get('--nsfw')) or "0")), - ("channelId", get_userinfo()), - ("videofile", get_file(options.get('--file'))) + ("videofile", get_videofile(path)) ] if options.get('--tags'): @@ -115,12 +151,21 @@ def upload_video(oauth, secret, options): fields.append(("thumbnailfile", get_file(options.get('--thumbnail')))) fields.append(("previewfile", get_file(options.get('--thumbnail')))) + if options.get('--playlist'): + playlist_id = get_playlist_by_name(user_info, options) + if not playlist_id and options.get('--playlistCreate'): + playlist_id = create_playlist(oauth, url, options) + else: + playlist_id = user_info['id'] + else: + playlist_id = user_info['id'] + fields.append(("channelId", str(playlist_id))) + multipart_data = MultipartEncoder(fields) headers = { 'Content-Type': multipart_data.content_type } - response = oauth.post(url + "/api/v1/videos/upload", data=multipart_data, headers=headers) From 9118f7b082c6825cca50c974dd7b3cb8d095b9d1 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 14:07:37 +0200 Subject: [PATCH 19/38] Add option to manage playlist for videos --- prismedia_upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/prismedia_upload.py b/prismedia_upload.py index 27d8b8d..04473d8 100755 --- a/prismedia_upload.py +++ b/prismedia_upload.py @@ -37,6 +37,10 @@ Options: --thumbnail=STRING Path to a file to use as a thumbnail for the video. Supported types are jpg and jpeg. By default, prismedia search for an image based on video name followed by .jpg or .jpeg + --playlist=STRING Set the playlist to use for the video. Also known as Channel for Peertube. + If the playlist is not found, spawn an error except if --playlist-create is set. + --playlistCreate Create the playlist if not exists. (default do not create) + Only relevant if --playlist is set. -h --help Show this help. --version Show version. @@ -211,6 +215,8 @@ if __name__ == '__main__': Optional('--thumbnail'): Or(None, And( str, validateThumbnail, error='thumbnail is not supported, please use jpg/jpeg'), ), + Optional('--playlist'): Or(None, str), + Optional('--playlistCreate'): bool, '--help': bool, '--version': bool }) From 461beaa5cb3516c1d1c0df55a5213f10aedee257 Mon Sep 17 00:00:00 2001 From: Zykino Date: Wed, 12 Sep 2018 00:45:18 +0200 Subject: [PATCH 20/38] peertube: fix default playlist --- lib/pt_upload.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index afbb8ff..2cc1083 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -48,8 +48,11 @@ def get_authenticated_service(secret): return oauth -def get_playlist_by_name(user_info, options): +def get_default_playlist(user_info): + return user_info['videoChannels'][0]['id'] + +def get_playlist_by_name(user_info, options): for playlist in user_info["videoChannels"]: if playlist['displayName'] == options.get('--playlist'): return playlist['id'] @@ -155,10 +158,11 @@ def upload_video(oauth, secret, options): playlist_id = get_playlist_by_name(user_info, options) if not playlist_id and options.get('--playlistCreate'): playlist_id = create_playlist(oauth, url, options) - else: - playlist_id = user_info['id'] + elif not playlist_id: + logging.warning("Playlist `" + options.get('--playlist') + "` is unknown, using default playlist.") + playlist_id = get_default_playlist(user_info) else: - playlist_id = user_info['id'] + playlist_id = get_default_playlist(user_info) fields.append(("channelId", str(playlist_id))) multipart_data = MultipartEncoder(fields) From 972cfd73d36d6ea652449979ab7d8cfff6fb546b Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 13:50:39 +0200 Subject: [PATCH 21/38] Update peeror repo with new url --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 713bf4c..9d25977 100644 --- a/README.md +++ b/README.md @@ -170,4 +170,4 @@ Languages: If your server uses peertube before 1.0.0-beta4, use the version inside tag 1.0.0-beta3! ## Sources -inspired by [peeror](https://git.drycat.fr/rigelk/Peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) +inspired by [peeror](https://git.rigelk.eu/rigelk/peeror) and [youtube-upload](https://github.com/tokland/youtube-upload) \ No newline at end of file From cb8ae77a1052a218298e678a92e0aa8977e0ac7f Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 14:07:02 +0200 Subject: [PATCH 22/38] Add support for playlist on Peertube --- lib/pt_upload.py | 54 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index a765de9..997f335 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -48,17 +48,54 @@ def get_authenticated_service(secret): return oauth +def get_playlist_by_name(user_info, options): + + for playlist in user_info["videoChannels"]: + if playlist['displayName'] == options.get('--playlist'): + return playlist['id'] + + +def create_playlist(oauth, url, options): + template = ('Peertube : Playlist %s does not exist, creating it.') + logging.info(template % (str(options.get('--playlist')))) + data = '{"displayName":"' + str(options.get('--playlist')) +'", \ + "description":null}' + + headers = { + 'Content-Type': "application/json" + } + try: + response = oauth.post(url + "/api/v1/video-channels/", + data=data, + headers=headers) + except Exception as e: + if hasattr(e, 'message'): + logging.error("Error: " + str(e.message)) + else: + logging.error("Error: " + str(e)) + if response is not None: + if response.status_code == 200: + jresponse = response.json() + jresponse = jresponse['videoChannel'] + return jresponse['id'] + else: + logging.error(('Peertube : The upload failed with an unexpected response: ' + '%s') % response) + exit(1) + + def upload_video(oauth, secret, options): def get_userinfo(): - user_info = json.loads(oauth.get(url + "/api/v1/users/me").content) - return str(user_info["id"]) + return json.loads(oauth.get(url+"/api/v1/users/me").content) def get_file(path): mimetypes.init() return (basename(path), open(abspath(path), 'rb'), mimetypes.types_map[splitext(path)[1]]) + path = options.get('--file') + user_info = get_userinfo() url = str(secret.get('peertube', 'peertube_url')).rstrip('/') # We need to transform fields into tuple to deal with tags as @@ -70,7 +107,6 @@ def upload_video(oauth, secret, options): ("licence", "1"), ("description", options.get('--description') or "default description"), ("nsfw", str(int(options.get('--nsfw')) or "0")), - ("channelId", get_userinfo()), ("videofile", get_file(options.get('--file'))) ] @@ -115,12 +151,21 @@ def upload_video(oauth, secret, options): fields.append(("thumbnailfile", get_file(options.get('--thumbnail')))) fields.append(("previewfile", get_file(options.get('--thumbnail')))) + if options.get('--playlist'): + playlist_id = get_playlist_by_name(user_info, options) + if not playlist_id and options.get('--playlistCreate'): + playlist_id = create_playlist(oauth, url, options) + else: + playlist_id = user_info['id'] + else: + playlist_id = user_info['id'] + fields.append(("channelId", str(playlist_id))) + multipart_data = MultipartEncoder(fields) headers = { 'Content-Type': multipart_data.content_type } - response = oauth.post(url + "/api/v1/videos/upload", data=multipart_data, headers=headers) @@ -138,7 +183,6 @@ def upload_video(oauth, secret, options): else: logging.error(('Peertube: The upload failed with an unexpected response: ' '%s') % response) - print(response.json()) exit(1) From 9efb18eb558a168c90e3643b48ad6bbfdeecf61a Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sat, 4 Aug 2018 14:07:37 +0200 Subject: [PATCH 23/38] Add option to manage playlist for videos --- prismedia_upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/prismedia_upload.py b/prismedia_upload.py index 4634440..68ed0a2 100755 --- a/prismedia_upload.py +++ b/prismedia_upload.py @@ -37,6 +37,10 @@ Options: --thumbnail=STRING Path to a file to use as a thumbnail for the video. Supported types are jpg and jpeg. By default, prismedia search for an image based on video name followed by .jpg or .jpeg + --playlist=STRING Set the playlist to use for the video. Also known as Channel for Peertube. + If the playlist is not found, spawn an error except if --playlist-create is set. + --playlistCreate Create the playlist if not exists. (default do not create) + Only relevant if --playlist is set. -h --help Show this help. --version Show version. @@ -211,6 +215,8 @@ if __name__ == '__main__': Optional('--thumbnail'): Or(None, And( str, validateThumbnail, error='thumbnail is not supported, please use jpg/jpeg'), ), + Optional('--playlist'): Or(None, str), + Optional('--playlistCreate'): bool, '--help': bool, '--version': bool }) From 95f6bc930f3be85ec57741ed30337dc439535a46 Mon Sep 17 00:00:00 2001 From: Zykino Date: Mon, 1 Oct 2018 21:24:34 +0200 Subject: [PATCH 24/38] fix merge: function name changed --- lib/pt_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index 2cc1083..4dd6143 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -110,7 +110,7 @@ def upload_video(oauth, secret, options): ("licence", "1"), ("description", options.get('--description') or "default description"), ("nsfw", str(int(options.get('--nsfw')) or "0")), - ("videofile", get_videofile(path)) + ("videofile", get_file(path)) ] if options.get('--tags'): From 2d7b8e0b095cf02fb332c79e8b2ec8536f2ec7a4 Mon Sep 17 00:00:00 2001 From: Zykino Date: Tue, 2 Oct 2018 12:11:26 +0200 Subject: [PATCH 25/38] Update README to show the current state of the playlist integration --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f612c76..9e5ee38 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,8 @@ Languages: - [x] thumbnail/preview - [x] multiple lines description (see [issue 4](https://git.lecygnenoir.info/LecygneNoir/prismedia/issues/4)) - [ ] add videos to playlist (YT & PT workflow: upload video, find playlist id, add video to playlist) + - [x] Peertube + - [ ] Youtube - [x] Use a config file (NFO) file to retrieve videos arguments - [x] Allow to choose peertube or youtube upload (to resume failed upload for example) - [x] Add publishAt option to plan your videos (need the [atd](https://linux.die.net/man/8/atd) daemon, [curl](https://linux.die.net/man/1/curl) and [jq](https://stedolan.github.io/jq/)) From 8d02d5a3a192c11081cb96b2002e588918d8adaf Mon Sep 17 00:00:00 2001 From: Zykino Date: Tue, 2 Oct 2018 00:30:33 +0200 Subject: [PATCH 26/38] Update peertube "publish at" functionnality to use their API --- lib/pt_upload.py | 26 ++++++++++++++++------ lib/utils.py | 56 ------------------------------------------------ 2 files changed, 19 insertions(+), 63 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index a765de9..b6a4626 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -5,7 +5,10 @@ import os import mimetypes import json import logging +import datetime +import pytz from os.path import splitext, basename, abspath +from tzlocal import get_localzone from ConfigParser import RawConfigParser from requests_oauthlib import OAuth2Session @@ -101,16 +104,27 @@ def upload_video(oauth, secret, options): # if no language, set default to 1 (English) fields.append(("language", "en")) - if options.get('--privacy'): - fields.append(("privacy", str(PEERTUBE_PRIVACY[options.get('--privacy').lower()]))) - else: - fields.append(("privacy", "3")) - if options.get('--disable-comments'): fields.append(("commentsEnabled", "0")) else: fields.append(("commentsEnabled", "1")) + privacy = None + if options.get('--privacy'): + privacy = options.get('--privacy').lower() + + if options.get('--publishAt'): + publishAt = options.get('--publishAt') + publishAt = datetime.datetime.strptime(publishAt, '%Y-%m-%dT%H:%M:%S') + tz = get_localzone() + tz = pytz.timezone(str(tz)) + publishAt = tz.localize(publishAt).isoformat() + fields.append(("scheduleUpdate[updateAt]", publishAt)) + fields.append(("scheduleUpdate[privacy]", str(PEERTUBE_PRIVACY["public"]))) + fields.append(("privacy", str(PEERTUBE_PRIVACY["private"]))) + else: + fields.append(("privacy", str(PEERTUBE_PRIVACY[privacy or "private"]))) + if options.get('--thumbnail'): fields.append(("thumbnailfile", get_file(options.get('--thumbnail')))) fields.append(("previewfile", get_file(options.get('--thumbnail')))) @@ -133,8 +147,6 @@ def upload_video(oauth, secret, options): logging.info('Peertube : Video was successfully uploaded.') template = 'Peertube: Watch it at %s/videos/watch/%s.' logging.info(template % (url, uuid)) - if options.get('--publishAt'): - utils.publishAt(str(options.get('--publishAt')), oauth, url, idvideo, secret) else: logging.error(('Peertube: The upload failed with an unexpected response: ' '%s') % response) diff --git a/lib/utils.py b/lib/utils.py index f1958e1..bfabaad 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -193,62 +193,6 @@ def parseNFO(options): def upcaseFirstLetter(s): return s[0].upper() + s[1:] - -def publishAt(publishAt, oauth, url, idvideo, secret): - try: - FNULL = open(devnull, 'w') - check_call(["at", "-V"], stdout=FNULL, stderr=STDOUT) - except CalledProcessError: - logging.error("You need to install the atd daemon to use the publishAt option.") - exit(1) - try: - FNULL = open(devnull, 'w') - check_call(["curl", "-V"], stdout=FNULL, stderr=STDOUT) - except CalledProcessError: - logging.error("You need to install the curl command line to use the publishAt option.") - exit(1) - try: - FNULL = open(devnull, 'w') - check_call(["jq", "-V"], stdout=FNULL, stderr=STDOUT) - except CalledProcessError: - logging.error("You need to install the jq command line to use the publishAt option.") - exit(1) - time = publishAt.split("T") - # Remove leading seconds that atd does not manage - if time[1].count(":") == 2: - time[1] = time[1][:-3] - - atTime = time[1] + " " + time[0] - refresh_token=str(oauth.__dict__['_client'].__dict__['refresh_token']) - atFile = "/tmp/peertube_" + idvideo + "_" + publishAt + ".at" - try: - openfile = open(atFile,"w") - openfile.write('token=$(curl -X POST -d "client_id=' + str(secret.get('peertube', 'client_id')) + - '&client_secret=' + str(secret.get('peertube', 'client_secret')) + - '&grant_type=refresh_token&refresh_token=' + str(refresh_token) + - '" "' + url + '/api/v1/users/token" | jq -r .access_token)') - openfile.write("\n") - openfile.write('curl "' + url + '/api/v1/videos/' + idvideo + - '" -X PUT -H "Authorization: Bearer ${token}"' + - ' -H "Content-Type: multipart/form-data" -F "privacy=1"') - openfile.write("\n ") # atd needs an empty line at the end of the file to load... - openfile.close() - except Exception as e: - if hasattr(e, 'message'): - logging.error("Error: " + str(e.message)) - else: - logging.error("Error: " + str(e)) - - try: - FNULL = open(devnull, 'w') - check_call(["at", "-M", "-f", atFile, atTime], stdout=FNULL, stderr=STDOUT) - except Exception as e: - if hasattr(e, 'message'): - logging.error("Error: " + str(e.message)) - else: - logging.error("Error: " + str(e)) - - def mastodonTag(tag): tags = tag.split(' ') mtag = '' From 5d59889f82f9994f3c021d17c657aca7a567b9c6 Mon Sep 17 00:00:00 2001 From: Zykino Date: Tue, 2 Oct 2018 11:21:20 +0200 Subject: [PATCH 27/38] Remove external utilities in README not used anymore for to publishAt --- README.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index 713bf4c..189dfd9 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,6 @@ Search in your package manager, otherwise use ``pip install --upgrade`` - requests-toolbelt - tzlocal -For Peertube and if you want to use the publishAt option, you also need some utilities on you local system - - [atd](https://linux.die.net/man/8/atd) daemon - - [curl](https://linux.die.net/man/1/curl) - - [jq](https://stedolan.github.io/jq/) - ## Configuration Edit peertube_secret and youtube_secret.json with your credentials. @@ -118,7 +113,6 @@ Options: --publishAt=DATE Publish the video at the given DATE using local server timezone. DATE should be on the form YYYY-MM-DDThh:mm:ss eg: 2018-03-12T19:00:00 DATE should be in the future - For Peertube, requires the "atd" and "curl utilities installed on the system --thumbnail=STRING Path to a file to use as a thumbnail for the video. Supported types are jpg and jpeg. By default, prismedia search for an image based on video name followed by .jpg or .jpeg @@ -160,7 +154,7 @@ Languages: - [ ] add videos to playlist (YT & PT workflow: upload video, find playlist id, add video to playlist) - [x] Use a config file (NFO) file to retrieve videos arguments - [x] Allow to choose peertube or youtube upload (to resume failed upload for example) -- [x] Add publishAt option to plan your videos (need the [atd](https://linux.die.net/man/8/atd) daemon, [curl](https://linux.die.net/man/1/curl) and [jq](https://stedolan.github.io/jq/)) +- [x] Add publishAt option to plan your videos - [ ] Record and forget: put the video in a directory, and the script uploads it for you - [ ] Usable on Desktop (Linux and/or Windows and/or MacOS) - [ ] Graphical User Interface From b4691bba1e3827ca4d6731da65c545876da5fce8 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Sun, 7 Oct 2018 21:26:21 +0200 Subject: [PATCH 28/38] Correct typo in readme for playlist feature --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 14cda0b..8c2b3d9 100644 --- a/README.md +++ b/README.md @@ -157,9 +157,8 @@ Languages: - [x] set default language - [x] thumbnail/preview - [x] multiple lines description (see [issue 4](https://git.lecygnenoir.info/LecygneNoir/prismedia/issues/4)) - - [ ] add videos to playlist (YT & PT workflow: upload video, find playlist id, add video to playlist) - - [x] Peertube - - [ ] Youtube + - [x] add videos to playlist for Peertube + - [ ] add videos to playlist for Youtube - [x] Use a config file (NFO) file to retrieve videos arguments - [x] Allow to choose peertube or youtube upload (to resume failed upload for example) - [x] Add publishAt option to plan your videos (need the [atd](https://linux.die.net/man/8/atd) daemon, [curl](https://linux.die.net/man/1/curl) and [jq](https://stedolan.github.io/jq/)) From d744ccb45eea4d670b826e621669004bbad679c4 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Mon, 15 Oct 2018 11:37:52 +0200 Subject: [PATCH 29/38] Update changelog for new publishAt feature --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a95b0d..12e2a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features - Add the possibility to upload thumbnail + - Use the API instead of external binaries for publishAt (thanks @zykino) ## v0.5 From 2565071e40dcfd3adaaa184181e0aacfd6bbc523 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Mon, 15 Oct 2018 12:42:09 +0200 Subject: [PATCH 30/38] Add examples for using playlist feature --- nfo_example.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nfo_example.txt b/nfo_example.txt index a4d4096..9a3890d 100644 --- a/nfo_example.txt +++ b/nfo_example.txt @@ -18,6 +18,8 @@ cca = True privacy = private disable-comments = True thumbnail = /path/to/your/thumbnail.jpg # Set the absolute path to your thumbnail +playlist = My Test Playlist +playlistCreate = True nsfw = True platform = youtube, peertube language = French From d3a42c4be1bc35218189ed69334748da00296537 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Mon, 15 Oct 2018 12:42:40 +0200 Subject: [PATCH 31/38] Patch playlist to be comaptible with beta16 (using name AND display name) --- lib/pt_upload.py | 11 ++++++----- lib/utils.py | 18 +++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index 4b57614..6e3a9cf 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -62,9 +62,10 @@ def get_playlist_by_name(user_info, options): def create_playlist(oauth, url, options): - template = ('Peertube : Playlist %s does not exist, creating it.') + template = ('Peertube: Playlist %s does not exist, creating it.') logging.info(template % (str(options.get('--playlist')))) - data = '{"displayName":"' + str(options.get('--playlist')) +'", \ + data = '{"name":"' + utils.cleanString(str(options.get('--playlist'))) +'", \ + "displayName":"' + str(options.get('--playlist')) +'", \ "description":null}' headers = { @@ -85,7 +86,7 @@ def create_playlist(oauth, url, options): jresponse = jresponse['videoChannel'] return jresponse['id'] else: - logging.error(('Peertube : The upload failed with an unexpected response: ' + logging.error(('Peertube: The upload failed with an unexpected response: ' '%s') % response) exit(1) @@ -101,7 +102,7 @@ def upload_video(oauth, secret, options): mimetypes.types_map[splitext(path)[1]]) path = options.get('--file') - url = secret.get('peertube', 'peertube_url') + url = str(secret.get('peertube', 'peertube_url')).rstrip('/') user_info = get_userinfo() # We need to transform fields into tuple to deal with tags as @@ -128,7 +129,7 @@ def upload_video(oauth, secret, options): exit(1) # If Mastodon compatibility is enabled, clean tags from special characters if options.get('--mt'): - strtag = utils.mastodonTag(strtag) + strtag = utils.cleanString(strtag) fields.append(("tags", strtag)) if options.get('--category'): diff --git a/lib/utils.py b/lib/utils.py index bfabaad..4234116 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -193,15 +193,15 @@ def parseNFO(options): def upcaseFirstLetter(s): return s[0].upper() + s[1:] -def mastodonTag(tag): - tags = tag.split(' ') - mtag = '' - for s in tags: +def cleanString(toclean): + toclean = toclean.split(' ') + cleaned = '' + for s in toclean: if s == '': continue - strtag = unicodedata.normalize('NFKD', unicode (s, 'utf-8')).encode('ASCII', 'ignore') - strtag = ''.join(e for e in strtag if e.isalnum()) - strtag = upcaseFirstLetter(strtag) - mtag = mtag + strtag + strtoclean = unicodedata.normalize('NFKD', unicode (s, 'utf-8')).encode('ASCII', 'ignore') + strtoclean = ''.join(e for e in strtoclean if e.isalnum()) + strtoclean = upcaseFirstLetter(strtoclean) + cleaned = cleaned + strtoclean - return mtag + return cleaned From 070408bb67c99fb130922f962e7dcd918bdf1a9b Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Thu, 18 Oct 2018 22:42:25 +0200 Subject: [PATCH 32/38] Add playlist feature to Youtube --- README.md | 2 +- lib/yt_upload.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 14c8a7e..189bb56 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Languages: - [x] thumbnail/preview - [x] multiple lines description (see [issue 4](https://git.lecygnenoir.info/LecygneNoir/prismedia/issues/4)) - [x] add videos to playlist for Peertube - - [ ] add videos to playlist for Youtube + - [x] add videos to playlist for Youtube - [x] Use a config file (NFO) file to retrieve videos arguments - [x] Allow to choose peertube or youtube upload (to resume failed upload for example) - [x] Add publishAt option to plan your videos diff --git a/lib/yt_upload.py b/lib/yt_upload.py index b28d495..0574aa1 100644 --- a/lib/yt_upload.py +++ b/lib/yt_upload.py @@ -51,13 +51,14 @@ RETRIABLE_STATUS_CODES = [500, 502, 503, 504] CLIENT_SECRETS_FILE = 'youtube_secret.json' CREDENTIALS_PATH = ".youtube_credentials.json" -SCOPES = ['https://www.googleapis.com/auth/youtube.upload'] +SCOPES = ['https://www.googleapis.com/auth/youtube.upload', 'https://www.googleapis.com/auth/youtube.force-ssl'] API_SERVICE_NAME = 'youtube' API_VERSION = 'v3' # Authorize the request and store authorization credentials. def get_authenticated_service(): + check_authenticated_scopes() flow = InstalledAppFlow.from_client_secrets_file( CLIENT_SECRETS_FILE, SCOPES) if exists(CREDENTIALS_PATH): @@ -121,6 +122,17 @@ def initialize_upload(youtube, options): publishAt = tz.localize(publishAt).isoformat() body['status']['publishAt'] = str(publishAt) + if options.get('--playlist'): + playlist_id = get_playlist_by_name(youtube, options.get('--playlist')) + if not playlist_id and options.get('--playlistCreate'): + playlist_id = create_playlist(youtube, options.get('--playlist')) + elif not playlist_id: + logging.warning("Youtube: Playlist `" + options.get('--playlist') + "` is unknown.") + logging.warning("If you want to create it, set the --playlistCreate option.") + playlist_id = "" + else: + playlist_id = "" + # Call the API's videos.insert method to create and upload the video. insert_request = youtube.videos().insert( part=','.join(body.keys()), @@ -133,9 +145,77 @@ def initialize_upload(youtube, options): if video_id and options.get('--thumbnail'): set_thumbnail(youtube, options.get('--thumbnail'), videoId=video_id) + # If we get a video_id, upload is successful and we are able to set playlist + if video_id and options.get('--playlist'): + set_playlist(youtube, playlist_id, video_id) + + +def get_playlist_by_name(youtube, playlist_name): + response = youtube.playlists().list( + part='snippet,id', + mine=True, + maxResults=50 + ).execute() + for playlist in response["items"]: + if playlist["snippet"]['title'] == playlist_name: + return playlist['id'] + + +def create_playlist(youtube, playlist_name): + template = ('Youtube: Playlist %s does not exist, creating it.') + logging.info(template % (str(playlist_name))) + resources = build_resource({'snippet.title': playlist_name, + 'snippet.description': '', + 'status.privacyStatus': 'public'}) + response = youtube.playlists().insert( + body=resources, + part='status,snippet,id' + ).execute() + return response["id"] + + +def build_resource(properties): + resource = {} + for p in properties: + # Given a key like "snippet.title", split into "snippet" and "title", where + # "snippet" will be an object and "title" will be a property in that object. + prop_array = p.split('.') + ref = resource + for pa in range(0, len(prop_array)): + is_array = False + key = prop_array[pa] + + # For properties that have array values, convert a name like + # "snippet.tags[]" to snippet.tags, and set a flag to handle + # the value as an array. + if key[-2:] == '[]': + key = key[0:len(key)-2:] + is_array = True + + if pa == (len(prop_array) - 1): + # Leave properties without values out of inserted resource. + if properties[p]: + if is_array: + ref[key] = properties[p].split(',') + else: + ref[key] = properties[p] + elif key not in ref: + # For example, the property is "snippet.title", but the resource does + # not yet have a "snippet" object. Create the snippet object here. + # Setting "ref = ref[key]" means that in the next time through the + # "for pa in range ..." loop, we will be setting a property in the + # resource's "snippet" object. + ref[key] = {} + ref = ref[key] + else: + # For example, the property is "snippet.description", and the resource + # already has a "snippet" object. + ref = ref[key] + return resource + def set_thumbnail(youtube, media_file, **kwargs): - kwargs = utils.remove_empty_kwargs(**kwargs) # See full sample for function + kwargs = utils.remove_empty_kwargs(**kwargs) request = youtube.thumbnails().set( media_body=MediaFileUpload(media_file, chunksize=-1, resumable=True), @@ -146,6 +226,26 @@ def set_thumbnail(youtube, media_file, **kwargs): return resumable_upload(request, 'thumbnail', 'set') +def set_playlist(youtube, playlist_id, video_id): + logging.info('Youtube: Configuring playlist...') + resource = build_resource({'snippet.playlistId': playlist_id, + 'snippet.resourceId.kind': 'youtube#video', + 'snippet.resourceId.videoId': video_id, + 'snippet.position': ''} + ) + try: + youtube.playlistItems().insert( + body=resource, + part='snippet' + ).execute() + except Exception as e: + if hasattr(e, 'message'): + logging.error("Youtube: Error: " + str(e.message)) + else: + logging.error("Youtube: Error: " + str(e)) + logging.info('Youtube: Video is correclty added to the playlist.') + + # This method implements an exponential backoff strategy to resume a # failed upload. def resumable_upload(request, resource, method): From 75aa36e1aa83b5f81f5f223e6e9f627dc615bfcd Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Thu, 18 Oct 2018 22:52:06 +0200 Subject: [PATCH 33/38] Check if scopes are corrects in youtube credentials --- lib/yt_upload.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/yt_upload.py b/lib/yt_upload.py index 0574aa1..82632a5 100644 --- a/lib/yt_upload.py +++ b/lib/yt_upload.py @@ -9,6 +9,7 @@ import time import copy import json from os.path import splitext, basename, exists +import os import google.oauth2.credentials import datetime import pytz @@ -80,6 +81,16 @@ def get_authenticated_service(): return build(API_SERVICE_NAME, API_VERSION, credentials=credentials, cache_discovery=False) +def check_authenticated_scopes(): + if exists(CREDENTIALS_PATH): + with open(CREDENTIALS_PATH, 'r') as f: + credential_params = json.load(f) + # Check if all scopes are present + if credential_params["_scopes"] != SCOPES: + logging.warning("Youtube: Credentials are obsolete, need to re-authenticate.") + os.remove(CREDENTIALS_PATH) + + def initialize_upload(youtube, options): path = options.get('--file') tags = None From 20d1ab6a112c38cf12f69c29782f56c7934b62d0 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Thu, 18 Oct 2018 22:53:45 +0200 Subject: [PATCH 34/38] Use console instead of local_server for authentication in Youtube to enable scripting --- lib/yt_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/yt_upload.py b/lib/yt_upload.py index 82632a5..daebdc9 100644 --- a/lib/yt_upload.py +++ b/lib/yt_upload.py @@ -73,7 +73,7 @@ def get_authenticated_service(): client_secret=credential_params["_client_secret"] ) else: - credentials = flow.run_local_server() + credentials = flow.run_console() with open(CREDENTIALS_PATH, 'w') as f: p = copy.deepcopy(vars(credentials)) del p["expiry"] From 745548abba0e11b2721edf37c22daf87ece55012 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Thu, 18 Oct 2018 23:08:56 +0200 Subject: [PATCH 35/38] Add -f as an alias for --file in command line, fix #16 --- prismedia_upload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prismedia_upload.py b/prismedia_upload.py index 68ed0a2..c5cbb8e 100755 --- a/prismedia_upload.py +++ b/prismedia_upload.py @@ -6,11 +6,12 @@ prismedia_upload - tool to upload videos to Peertube and Youtube Usage: prismedia_upload.py --file= [options] - prismedia_upload.py --file= --tags=STRING [--mt options] + prismedia_upload.py -f --tags=STRING [--mt options] prismedia_upload.py -h | --help prismedia_upload.py --version Options: + -f, --file=STRING Path to the video file to upload in mp4 --name=NAME Name of the video to upload. (default to video filename) -d, --description=STRING Description of the video. (default: default description) -t, --tags=STRING Tags for the video. comma separated. From 81a183dd729772adf11762151b8c52663af66aaa Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Fri, 19 Oct 2018 13:48:32 +0200 Subject: [PATCH 36/38] fix max lenght for playlist name in Peertube --- lib/pt_upload.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pt_upload.py b/lib/pt_upload.py index 6e3a9cf..25b9e0c 100644 --- a/lib/pt_upload.py +++ b/lib/pt_upload.py @@ -64,7 +64,10 @@ def get_playlist_by_name(user_info, options): def create_playlist(oauth, url, options): template = ('Peertube: Playlist %s does not exist, creating it.') logging.info(template % (str(options.get('--playlist')))) - data = '{"name":"' + utils.cleanString(str(options.get('--playlist'))) +'", \ + playlist_name = utils.cleanString(str(options.get('--playlist'))) + # Peertube allows 20 chars max for playlist name + playlist_name = playlist_name[:19] + data = '{"name":"' + playlist_name +'", \ "displayName":"' + str(options.get('--playlist')) +'", \ "description":null}' From 041a8fd722ade99bb77a6019f9e3e62e960b41e5 Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Fri, 19 Oct 2018 14:48:52 +0200 Subject: [PATCH 37/38] Prepare documentation and changelog for release v0.6 --- CHANGELOG.md | 14 +++++++++++--- README.md | 14 ++++++-------- prismedia_upload.py | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12e2a1e..b99f101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,18 @@ # Changelog -## v 0.? +## v 0.6 + +### Compatibility ### +**Beware**, the first launch of prismedia for youtube will reask for credentials, this is needed for playlists. + +This release is fully compatible with Peertube v1.0.0! ### Features - - Add the possibility to upload thumbnail - - Use the API instead of external binaries for publishAt (thanks @zykino) + - Add the possibility to upload thumbnail. + - Add the possibility to configure playlist. (thanks @zykino for Peertube part) + - Use the API instead of external binaries for publishAt for both Peertube and Youtube. (thanks @zykino) + - Use the console option to authenticate against youtube for easier use with ssh'ed servers + - Add -f as an alias for --file for easier upload. ## v0.5 diff --git a/README.md b/README.md index 189bb56..cd712e9 100644 --- a/README.md +++ b/README.md @@ -82,15 +82,8 @@ Use a NFO file to specify your video options: Use --help to get all available options: ``` -prismedia_upload - tool to upload videos to Peertube and Youtube - -Usage: - prismedia_upload.py --file= [options] - prismedia_upload.py --file= --tags=STRING [--mt options] - prismedia_upload.py -h | --help - prismedia_upload.py --version - Options: + -f, --file=STRING Path to the video file to upload in mp4 --name=NAME Name of the video to upload. (default to video filename) -d, --description=STRING Description of the video. (default: default description) -t, --tags=STRING Tags for the video. comma separated. @@ -113,9 +106,14 @@ Options: --publishAt=DATE Publish the video at the given DATE using local server timezone. DATE should be on the form YYYY-MM-DDThh:mm:ss eg: 2018-03-12T19:00:00 DATE should be in the future + For Peertube, requires the "atd" and "curl utilities installed on the system --thumbnail=STRING Path to a file to use as a thumbnail for the video. Supported types are jpg and jpeg. By default, prismedia search for an image based on video name followed by .jpg or .jpeg + --playlist=STRING Set the playlist to use for the video. Also known as Channel for Peertube. + If the playlist is not found, spawn an error except if --playlist-create is set. + --playlistCreate Create the playlist if not exists. (default do not create) + Only relevant if --playlist is set. -h --help Show this help. --version Show version. diff --git a/prismedia_upload.py b/prismedia_upload.py index c5cbb8e..c11f08b 100755 --- a/prismedia_upload.py +++ b/prismedia_upload.py @@ -94,7 +94,7 @@ except ImportError: 'see https://github.com/ahupp/python-magic\n') exit(1) -VERSION = "prismedia v0.5" +VERSION = "prismedia v0.6" VALID_PRIVACY_STATUSES = ('public', 'private', 'unlisted') VALID_CATEGORIES = ( From bb61725e62b3cfdda093a792f4515b1e10e4a62f Mon Sep 17 00:00:00 2001 From: LecygneNoir Date: Fri, 19 Oct 2018 14:49:10 +0200 Subject: [PATCH 38/38] Prepare documentation and changelog for release v0.6 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b99f101..9ad15df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## v 0.6 +## v0.6 ### Compatibility ### **Beware**, the first launch of prismedia for youtube will reask for credentials, this is needed for playlists.