diff --git a/poetry.lock b/poetry.lock index 58c55fe..91ba0d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,17 +1,3 @@ -[[package]] -name = "asgiref" -version = "3.4.1" -description = "ASGI specs, helper code, and adapters" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -typing-extensions = {version = "*", markers = "python_version < \"3.8\""} - -[package.extras] -tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] - [[package]] name = "atomicwrites" version = "1.4.0" @@ -56,7 +42,7 @@ python-versions = "*" [[package]] name = "charset-normalizer" -version = "2.0.10" +version = "2.0.11" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false @@ -124,14 +110,6 @@ url = "https://github.com/dblueai/giteapy.git" reference = "master" resolved_reference = "001e9b66795a6d34146c8532e9d8e648d5b93e59" -[[package]] -name = "h11" -version = "0.12.0" -description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "main" -optional = false -python-versions = ">=3.6" - [[package]] name = "idna" version = "3.3" @@ -142,7 +120,7 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "4.10.0" +version = "4.10.1" description = "Read metadata from Python packages" category = "main" optional = false @@ -239,7 +217,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pyparsing" -version = "3.0.6" +version = "3.0.7" description = "Python parsing module" category = "dev" optional = false @@ -272,11 +250,11 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm [[package]] name = "pytest-mock" -version = "3.6.1" +version = "3.7.0" description = "Thin-wrapper around the mock package for easier use with pytest" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] pytest = ">=5.0" @@ -407,23 +385,6 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "uvicorn" -version = "0.16.0" -description = "The lightning-fast ASGI server." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -asgiref = ">=3.4.0" -click = ">=7.0" -h11 = ">=0.8" -typing-extensions = {version = "*", markers = "python_version < \"3.8\""} - -[package.extras] -standard = ["httptools (>=0.2.0,<0.4.0)", "watchgod (>=0.6)", "python-dotenv (>=0.13)", "PyYAML (>=5.1)", "websockets (>=9.1)", "websockets (>=10.0)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "colorama (>=0.4)"] - [[package]] name = "werkzeug" version = "2.0.2" @@ -450,13 +411,9 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "9a677f1eb74519bb45234f3aa90205f27449ebffa77c6ff509c3c5d69db3274f" +content-hash = "ea26a2a723bb4541c447d3cdab6d4c16dd941875e55d770300a134f39a21adc4" [metadata.files] -asgiref = [ - {file = "asgiref-3.4.1-py3-none-any.whl", hash = "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"}, - {file = "asgiref-3.4.1.tar.gz", hash = "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9"}, -] atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, @@ -474,8 +431,8 @@ certifi = [ {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.0.10.tar.gz", hash = "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd"}, - {file = "charset_normalizer-2.0.10-py3-none-any.whl", hash = "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455"}, + {file = "charset-normalizer-2.0.11.tar.gz", hash = "sha256:98398a9d69ee80548c762ba991a4728bfc3836768ed226b3945908d1a688371c"}, + {file = "charset_normalizer-2.0.11-py3-none-any.whl", hash = "sha256:2842d8f5e82a1f6aa437380934d5e1cd4fcf2003b06fed6940769c164a480a45"}, ] click = [ {file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"}, @@ -490,17 +447,13 @@ flask = [ {file = "Flask-2.0.2.tar.gz", hash = "sha256:7b2fb8e934ddd50731893bdcdb00fc8c0315916f9fcd50d22c7cc1a95ab634e2"}, ] giteapy = [] -h11 = [ - {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, - {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, -] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.10.0-py3-none-any.whl", hash = "sha256:b7cf7d3fef75f1e4c80a96ca660efbd51473d7e8f39b5ab9210febc7809012a4"}, - {file = "importlib_metadata-4.10.0.tar.gz", hash = "sha256:92a8b58ce734b2a4494878e0ecf7d79ccd7a128b5fc6014c401e0b61f006f0f6"}, + {file = "importlib_metadata-4.10.1-py3-none-any.whl", hash = "sha256:899e2a40a8c4a1aec681feef45733de8a6c58f3f6a0dbed2eb6574b4387a77b6"}, + {file = "importlib_metadata-4.10.1.tar.gz", hash = "sha256:951f0d8a5b7260e9db5e41d429285b5f451e928479f19d80818878527d36e95e"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -602,16 +555,16 @@ pycodestyle = [ {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, ] pyparsing = [ - {file = "pyparsing-3.0.6-py3-none-any.whl", hash = "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4"}, - {file = "pyparsing-3.0.6.tar.gz", hash = "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"}, + {file = "pyparsing-3.0.7-py3-none-any.whl", hash = "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"}, + {file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"}, ] pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] pytest-mock = [ - {file = "pytest-mock-3.6.1.tar.gz", hash = "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62"}, - {file = "pytest_mock-3.6.1-py3-none-any.whl", hash = "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3"}, + {file = "pytest-mock-3.7.0.tar.gz", hash = "sha256:5112bd92cc9f186ee96e1a92efc84969ea494939c3aead39c50f421c4cc69534"}, + {file = "pytest_mock-3.7.0-py3-none-any.whl", hash = "sha256:6cff27cec936bf81dc5ee87f07132b807bcda51106b5ec4b90a04331cba76231"}, ] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, @@ -688,10 +641,6 @@ urllib3 = [ {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, ] -uvicorn = [ - {file = "uvicorn-0.16.0-py3-none-any.whl", hash = "sha256:d8c839231f270adaa6d338d525e2652a0b4a5f4c2430b5c4ef6ae4d11776b0d2"}, - {file = "uvicorn-0.16.0.tar.gz", hash = "sha256:eacb66afa65e0648fcbce5e746b135d09722231ffffc61883d4fac2b62fbea8d"}, -] werkzeug = [ {file = "Werkzeug-2.0.2-py3-none-any.whl", hash = "sha256:63d3dc1cf60e7b7e35e97fa9861f7397283b75d765afcaefd993d6046899de8f"}, {file = "Werkzeug-2.0.2.tar.gz", hash = "sha256:aa2bb6fc8dee8d6c504c0ac1e7f5f7dc5810a9903e793b6f715a9f015bdadb9a"}, diff --git a/pyproject.toml b/pyproject.toml index 1ee007e..b0dbcf6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ packages = [ [tool.poetry.dependencies] python = "^3.7.1" Flask = "^2.0.2" -giteapy = {git = "https://github.com/dblueai/giteapy.git"} +giteapy = {url = "https://github.com/dblueai/giteapy/archive/master.zip"} requests = "^2.27.1" python-dotenv = "^0.19.2" python-slugify = "^5.0.2" diff --git a/src/microcosm/__init__.py b/src/microcosm/__init__.py index 31db7fe..c92117b 100644 --- a/src/microcosm/__init__.py +++ b/src/microcosm/__init__.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union import requests import os import functools @@ -18,11 +18,9 @@ from slugify import slugify from datetime import date, datetime from xml.etree import ElementTree -from flask import Flask, request, url_for, Response +from flask import Flask, jsonify, request, url_for, Response from requests import api - - dotenv.load_dotenv() PERMITTED_DOMAIN = os.environ.get( @@ -81,31 +79,43 @@ def process_photo_url(now: datetime, doc: Dict[str, List[str]], suffix: str = "" """Process photo submitted via URL""" now_ts = int(time.mktime(now.timetuple())) + + photo_urls = [] - if os.environ.get('MICROPUB_IMAGE_STRATEGY') == 'copy': - # download the photo - r = requests.get(doc['photo']) + if isinstance(doc['photo'], str): + doc['photo'] = [doc['photo']] - ext = os.path.splitext(doc['photo'])[1] + + for i, photo in enumerate(doc['photo']): - # generate local filename - filename = os.path.join(os.environ.get( - 'MICROPUB_MEDIA_PATH'), now.strftime("%Y/%m/%d"), str(now_ts) + f"{now_ts}_{suffix}{ext}") - photo_url = os.path.join(os.environ.get( - 'MICROPUB_MEDIA_URL_PREFIX'), now.strftime("%Y/%m/%d"), str(now_ts) + f"{now_ts}_{suffix}{ext}") + if os.environ.get('MICROPUB_IMAGE_STRATEGY') == 'copy': + # download the photo - # make directory if needed - if not os.path.exists(os.path.dirname(filename)): - os.makedirs(os.path.dirname(filename)) + r = requests.get(photo) - with open(filename, 'wb') as f: - f.write(r.content) + ext = os.path.splitext(photo)[1] + + # generate local filename + filename = os.path.join(os.environ.get( + 'MICROPUB_MEDIA_PATH'), now.strftime("%Y/%m/%d"), str(now_ts) + f"{now_ts}_{suffix}_{i}_{ext}") + photo_url = os.path.join(os.environ.get( + 'MICROPUB_MEDIA_URL_PREFIX'), now.strftime("%Y/%m/%d"), str(now_ts) + f"{now_ts}_{suffix}_{i}_{ext}") + + photo_urls.append(photo_url) + + # make directory if needed + if not os.path.exists(os.path.dirname(filename)): + os.makedirs(os.path.dirname(filename)) + + with open(filename, 'wb') as f: + f.write(r.content) - else: - photo_url = doc['photo'] + else: + photo_urls.append(photo) + - return photo_url + return photo_urls def process_photo_upload(now: datetime, file: FileStorage, suffix: str=""): """Process photo directly uploaded to micropub""" @@ -139,7 +149,10 @@ def init_frontmatter(now: datetime, post_type: str, name: Optional[str]=None): now_ts = int(time.mktime(now.timetuple())) if name: - slug = slugify(name) + str(now_ts) + if isinstance(name, list): + slug = slugify(name[0]) + str(now_ts) + else: + slug = slugify(name) + str(now_ts) else: slug = str(now_ts) @@ -147,6 +160,8 @@ def init_frontmatter(now: datetime, post_type: str, name: Optional[str]=None): url = os.path.join("/", ENTITY_TYPE_PLURAL_MAP.get(post_type, post_type + "s"), now.strftime("%Y/%m/%d"), slug) + print(os.environ.get( + 'CONTENT_PREFIX')) file_path = os.path.join(os.environ.get( 'CONTENT_PREFIX'), ENTITY_TYPE_PLURAL_MAP.get(post_type, post_type + "s"), now.strftime("%Y/%m/%d"), slug + ".md") @@ -166,6 +181,8 @@ def webmention_hook(): body = request.get_json() + + print(f"Incoming webmention {body}") # webmention should always have a json body if body is None: @@ -218,7 +235,7 @@ def webmention_hook(): # format the mention like the data from webmention.io api new_mention = { - "id": body['wm-id'], + "id": body['post']['wm-id'], "source": body['source'], "target": body['target'], "activity":{ @@ -265,46 +282,70 @@ def webmention_hook(): except Exception as e: return {"error": str(e)}, 500 +def detect_entry_type(doc: dict) -> str: + """Given a dictionary object from either form or json, detect type of post""" + + + if ('in-reply-to' in doc) or ('u-in-reply-to' in doc): + entry_type = "reply" + + elif ('bookmark-of' in doc) or ('u-bookmark-of' in doc): + entry_type = "bookmark" + elif ('repost-of' in doc) or ('u-repost-of' in doc): + entry_type = "repost" + elif ('like-of' in doc) or ('u-like-of' in doc): + entry_type = "like" + + elif ('name' in doc) or ('p-name' in doc): + entry_type = "post" + else: + entry_type = "note" + + return entry_type + +def capture_frontmatter_props(doc: Dict[str, Union[str, List[str]]], frontmatter: Dict[str, Union[str,List[str]]]): + + + keys = ['bookmark-of', 'in-reply-to', 'repost-of', 'like-of'] + + keys += [f'u-{key}' for key in keys] + + for key in keys: + + if key in doc: + if isinstance(doc[key], list) and (len(doc[key]) < 2): + frontmatter[key] = doc[key][0] + else: + frontmatter[key] = doc[key] + + if 'category' in doc: + if isinstance(doc['category'], list): + categories = doc['category'] + else: + categories = [doc['category']] + elif 'p-category' in doc: + categories = doc['p-category'] + else: + categories = request.form.getlist('category[]') + + if len(categories) > 0: + frontmatter['tags'] = categories def process_multipart_post(): doc = request.form.to_dict(flat=True) - if 'in-reply-to' in doc: - entry_type = "reply" - - elif 'bookmark-of' in doc: - entry_type = "bookmark" - elif 'repost-of' in doc: - entry_type = "repost" - elif 'like-of' in doc: - entry_type = "like" - - elif 'name' in doc: - entry_type = "post" - else: - entry_type = "note" + entry_type = detect_entry_type(doc) now = datetime.now() frontmatter, file_path = init_frontmatter(now, entry_type, doc.get('name')) + capture_frontmatter_props(doc, frontmatter) - for key in ['bookmark-of', 'in-reply-to', 'repost-of', 'like-of']: - if key in doc: - frontmatter[key] = doc[key] - - if 'category' in doc: - categories = [doc['category']] - else: - categories = request.form.getlist('category[]') - if 'name' in doc: frontmatter['title'] = doc['name'] - if len(categories) > 0: - frontmatter['tags'] = categories - if ('photo' in doc) or ('photo' in request.files) or ('photo[]' in request.files): @@ -327,14 +368,14 @@ def process_multipart_post(): else: if 'photo' in doc: - photo_url = process_photo_url(now, doc) + photo_urls = process_photo_url(now, doc) else: - photo_url = process_photo_upload(now, request.files['photo']) + photo_urls = [process_photo_upload(now, request.files['photo'])] - frontmatter['photo'] = [photo_url] - - - docstr = f" \n\n {doc['content']}" + frontmatter['photo'] = photo_urls + docstr = "" + for photo in photo_urls: + docstr += f" \n\n {doc['content']}" else: docstr = doc.get('content','') if 'content' in doc else "" @@ -352,9 +393,52 @@ def process_multipart_post(): return docstr, frontmatter, file_path + def process_json_post(): """Process JSON POST submission""" + body = request.get_json() + + # get post type - take the first item in the array + if body['type'][0] != 'h-entry': + return jsonify({"error":"invalid_format"}), 400 + + props = body['properties'] + entry_type = detect_entry_type(props) + + now = datetime.now() + + frontmatter, file_path = init_frontmatter(now, entry_type, props.get('name')) + + capture_frontmatter_props(props, frontmatter) + + if 'name' in props: + frontmatter['title'] = props['name'][0] + + + docstr = "" + + if 'photo' in props: + + photo_urls = process_photo_url(now, props) + + frontmatter['photo'] = photo_urls + + for photo in photo_urls: + docstr += f"\n\n" + + for content in props.get('content', []): + + if isinstance(content, dict): + if 'html' in content: + docstr += f"\n\n {content.get('html')}" + + else: + docstr += f"\n\n {content}" + + return docstr, frontmatter, file_path + + @app.route('/', methods=['POST']) @@ -471,10 +555,5 @@ def index(): return {"syndicate-to": get_syndication_targets()} - - - - - if __name__ == '__main__': app.run(debug=True)