63 lines
1.6 KiB
Python
63 lines
1.6 KiB
Python
|
import click
|
||
|
|
import dotenv
|
||
|
|
import os
|
||
|
|
import requests
|
||
|
|
import ujson
|
||
|
|
from urllib.parse import urlparse
|
||
|
|
|
||
|
|
@click.group()
|
||
|
|
def cli():
|
||
|
|
dotenv.load_dotenv()
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
@cli.command()
|
||
|
|
@click.option("--mentions-file", type=click.Path(file_okay=True), required=True)
|
||
|
|
def fetch_mentions(mentions_file: str):
|
||
|
|
"""Fetch web mentions and store as json"""
|
||
|
|
|
||
|
|
|
||
|
|
mention_ids = set()
|
||
|
|
|
||
|
|
if os.path.exists(mentions_file):
|
||
|
|
print(f"Load existing mentions from {mentions_file}")
|
||
|
|
with open(mentions_file,'r') as f:
|
||
|
|
mentions = ujson.load(f)
|
||
|
|
print(mentions.keys())
|
||
|
|
print(f"Found existing mentions for {len(mentions.keys())} urls")
|
||
|
|
else:
|
||
|
|
mentions = {}
|
||
|
|
|
||
|
|
for mentionset in mentions.values():
|
||
|
|
mention_ids.update([post['id'] for post in mentionset])
|
||
|
|
|
||
|
|
print("Requesting new mentions...")
|
||
|
|
r = requests.get(f"https://webmention.io/api/mentions.json?token={os.environ.get('WEBMENTIONSIO_API_KEY')}")
|
||
|
|
|
||
|
|
if r.json().get('error') is not None:
|
||
|
|
print(f"Failed to request webmentions: {r.json()}")
|
||
|
|
return 1
|
||
|
|
|
||
|
|
new = 0
|
||
|
|
for link in r.json()['links']:
|
||
|
|
target = urlparse(link['target']).path
|
||
|
|
|
||
|
|
if target not in mentions:
|
||
|
|
mentions[target] = []
|
||
|
|
|
||
|
|
|
||
|
|
if link['id'] not in mention_ids:
|
||
|
|
mention_ids.add(link['id'])
|
||
|
|
mentions[target].append(link)
|
||
|
|
new += 1
|
||
|
|
|
||
|
|
print(f"Found {new} new mentions")
|
||
|
|
|
||
|
|
print(f"Storing mentions at {mentions_file}")
|
||
|
|
with open(mentions_file,'w') as f:
|
||
|
|
ujson.dump(mentions, f, indent=2)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
cli()
|