From cee043b6d2e229ae3171e698cb65265e0d2dfc9d Mon Sep 17 00:00:00 2001 From: nojhan Date: Sat, 4 Apr 2026 14:02:06 +0200 Subject: [PATCH 01/13] refactor the project up to modern standards --- pyproject.toml | 23 ++++++++++ forthlift.py => src/forthlift/forthlift.py | 52 ++++++++++++---------- 2 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 pyproject.toml rename forthlift.py => src/forthlift/forthlift.py (87%) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5e90228 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "forthlift" +version = "0.1.0" +description = "A command line application to post sequences of text lines on social media" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "argparse>=1.4.0", + "configparser>=7.2.0", + "datetime>=6.0", +] + +[project.scripts] +forthlift = "forthlift.forthlift:main" + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["forthlift*"] + diff --git a/forthlift.py b/src/forthlift/forthlift.py similarity index 87% rename from forthlift.py rename to src/forthlift/forthlift.py index b48ecaf..f66a736 100755 --- a/forthlift.py +++ b/src/forthlift/forthlift.py @@ -2,11 +2,12 @@ import sys import locale +import logging import argparse import datetime -import ConfigParser +from configparser import ConfigParser -import tweepy +#import tweepy class AppKeyError(Exception): @@ -160,24 +161,24 @@ def setup_twitter(configfile="twitter.conf"): app_key = config.get("App","app_key") app_secret = config.get("App","app_key_secret") - auth = tweepy.OAuthHandler(app_key, app_secret) + # auth = tweepy.OAuthHandler(app_key, app_secret) # Authenticate the user. - auth_url = auth.get_authorization_url() - print "Copy and paste this URL in your browser while your are logged in the twitter account where you want to post." - print "Authorization URL: " + auth_url + # auth_url = auth.get_authorization_url() + # print("Copy and paste this URL in your browser while your are logged in the twitter account where you want to post.") + # print("Authorization URL: " + auth_url) - print "Then paste the Personal Identification Number given by Twitter:" - verifier = raw_input('PIN: ').strip() - token = auth.get_access_token(verifier) + # print("Then paste the Personal Identification Number given by Twitter:") + # verifier = raw_input('PIN: ').strip() + # token = auth.get_access_token(verifier) # Authenticate and get the user name. - token_key, token_secret = token - auth.set_access_token(token_key, token_secret) - api = tweepy.API(auth) - username = api.me().name - print "Authentication successful, ready to post to account: " + username + # token_key, token_secret = token + # auth.set_access_token(token_key, token_secret) + # api = tweepy.API(auth) + # username = api.me().name + # print("Authentication successful, ready to post to account: " + username) # Save authentication tokens. @@ -200,8 +201,7 @@ def setup_twitter(configfile="twitter.conf"): # CLI # -if __name__=="__main__": - +def main(): errors = {"NO_ERROR":0, "UNKNOWN_ERROR":1, "NO_SETUP_NEEDED":2, "NO_APP_KEY":10} usage = "Post text read on the standard input to a website or an API." @@ -261,7 +261,7 @@ if __name__=="__main__": sys.stderr.write(e.msg) sys.exit(errors["NO_APP_KEY"]) except: - print "Unexpected error:", sys.exc_info()[0] + logging.error(f"Unexpected error:\n{sys.exc_info()[0]}") raise else: sys.exit(errors["NO_ERROR"]) @@ -294,9 +294,9 @@ if __name__=="__main__": except OSError: cannot.append(img) if cannot: - print("Cannot open the following image files, I will not continue: ") + logger.error("Cannot open the following image files, I will not continue: ") for img in cannot: - print(img) + logger.error(img) sys.exit(5) # I/O Error # APIs @@ -321,11 +321,15 @@ if __name__=="__main__": access_token = config.get("Auth","local_token") access_token_secret = config.get("Auth","local_token_secret") - auth = tweepy.OAuthHandler(app_key, app_secret, "https://api.twitter.com/1.1/") - auth.set_access_token(access_token, access_token_secret) + # auth = tweepy.OAuthHandler(app_key, app_secret, "https://api.twitter.com/1.1/") + # auth.set_access_token(access_token, access_token_secret) - api = tweepy.API(auth) + # api = tweepy.API(auth) - # Post - operate( on_twitter, api, asked ) + # # Post + # operate( on_twitter, api, asked ) + + +if __name__ == "__main__": + main() From 844209e7b652a84c2c08d68d3afad43aeb5aca17 Mon Sep 17 00:00:00 2001 From: nojhan Date: Sat, 4 Apr 2026 21:29:44 +0200 Subject: [PATCH 02/13] modular reimplementation from scratch --- src/forthlift/forthlift.py | 406 ++++++++++++------------------------- 1 file changed, 128 insertions(+), 278 deletions(-) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index f66a736..6b2ebbb 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -7,327 +7,177 @@ import argparse import datetime from configparser import ConfigParser -#import tweepy - - -class AppKeyError(Exception): - def __init__(self, msg): - self.msg = msg - - -def write(data, stream = sys.stdout): - """ - Write "data" on the given stream, then flush, silently handling broken pipes. - """ - try: - stream.write(data.encode(locale.getpreferredencoding(True) or "utf-8")) - stream.flush() - - # Silently handle broken pipes - except IOError: - try: - stream.close() - except IOError: - pass - - -def readline( stream_in ): - while True: - try: - line = stream_in.readline().decode(stream_in.encoding or locale.getpreferredencoding(True)).strip() - except UnicodeDecodeError: - continue - except KeyboardInterrupt: - break - if not line: - break - yield line - - raise StopIteration - - -def setup_counter( data, asked ): - assert(asked.counter) - - # unroll everything - data = list(data) - nb = len(data) - # " int/int" - counter_size = len(str(nb).encode("utf-8").decode("utf-8") )*2 + 1 + 1 - - for i,line in enumerate(data): - counter = u" %i/%i" % (i+1,nb) - curmax = asked.max_len - len(counter) - if len(line) > curmax: - if asked.ignore: - data[i] = line - elif asked.trim: - data[i] = line[:curmax] - data[i] = data[i] + counter - - return data - - -def setup_hem(data, asked): - for line in data: - if len(line) > asked.max_len: - if asked.ignore: - pass - elif asked.trim: - line = line[:asked.max_len] - - yield line - - -def setup( data, asked ): - if asked.counter: - f = setup_counter - else: - f = setup_hem - - for line in f(data, asked): - yield line - - -def operate( func, *args ): - for line in func( readline(sys.stdin), *args ): - write(line) - - # -# STDOUT API +# LIBRARY # -def on_stdout( data, asked, endline="\n" ): - lines = setup(data, asked) - # You can do something on the whole set of lines if you want. - for i,line in enumerate(lines): - if endline: - if line[-1] != endline: - line += endline - - if asked.independent: - yield line - else: - l = u"%i %s" % (i,line) - yield l +logger = logging.getLogger("forthlift") -# -# TWITTER API -# +class stream: + class Stream: + def __call__(self): + raise NotImplementedError -def on_twitter( data, api, asked, endline="\n" ): - lines = setup(data, asked) + class stdin(Stream): + def __call__(self): + return sys.stdin - prev_status_id = None +class consume: + class Consume: + def __call__(self): + raise NotImplementedError - images = asked.twitter_images - if images: - images.reverse() + class lines(Consume): + def __call__(self, stream): + return stream.readlines() - for line in lines: - if images: - img = images.pop() - else: - img = None +class format: + class Format: + def __call__(self, lines): + raise NotImplementedError - if img: - # API.update_with_media(filename[, status][, in_reply_to_status_id][, lat][, long][, source][, place_id][, file]) - status = api.update_with_media(img, line, prev_status_id) - else: - # API.update_status(status[, in_reply_to_status_id][, lat][, long][, source][, place_id]) - status = api.update_status(line, prev_status_id) + class as_is(Format): + def __call__(self, lines): + return lines - if asked.independent: - prev_status_id = None - else: - prev_status_id = status.id +class lift: + class Lift: + def __call__(self, items): + raise NotImplementedError - yield status.text + endline + class stdout(Lift): + def __call__(self, items): + for item in items: + print(item, end="") -def setup_twitter(configfile="twitter.conf"): +class Forthlifter: + def __init__(self, + consumer = consume.lines(), + streamers = [stream.stdin()], + formatters = [format.as_is()], + lifters = [lift.stdout()], + ): + self.consumer = consumer + logger.debug(f"├ consumer: {type(consumer).__name__}") - config = ConfigParser.RawConfigParser() + if not isinstance(streamers, list): + streamers = [streamers] + self.streamers = streamers + logger.debug(f"├ streamers: {', '.join(type(i).__name__ for i in self.streamers)}") - # Authenticate the application. - config.read(configfile) + if not isinstance(formatters, list): + formatters = [formatters] + self.formatters = formatters + logger.debug(f"├ formatters: {', '.join(type(i).__name__ for i in self.formatters)}") - if not config.has_section("App"): - raise AppKeyError("ERROR: did not found application keys, ask your distribution maintainer or get keys from Twitter first.") + if not isinstance(lifters, list): + lifters = [lifters] + self.lifters = lifters + logger.debug(f"└lifters: {', '.join(type(i).__name__ for i in self.lifters)}") - app_key = config.get("App","app_key") - app_secret = config.get("App","app_key_secret") + def consume(self): + logger.debug("│ ├ consume") + lines = [] + for streamer in self.streamers: + logger.debug(f"│ │ ├ {type(self.consumer).__name__}({type(streamer).__name__})") + # Concatenate + lines += self.consumer(streamer()) + logger.debug(f"│ │ ├ {len(lines)} lines") + logger.debug("│ │ └OK") + return lines - # auth = tweepy.OAuthHandler(app_key, app_secret) + def format(self, lines): + logger.debug(f"│ ├ format {len(lines)} lines") + items = lines + for formatter in self.formatters: + logger.debug(f"│ │ ├ {type(formatter).__name__}") + # Replace + items = formatter(items) + logger.debug(f"│ │ ├ {len(items)} items") + logger.debug(f"│ │ └OK {len(items)} items") + return items + def lift(self, items): + logger.debug(f"│ ├ lift {len(items)} items") + for lifter in self.lifters: + logger.debug(f"│ │ ├ {type(lifter).__name__}") + # Call + lifter(items) + logger.debug("│ │ └OK") - # Authenticate the user. - # auth_url = auth.get_authorization_url() - # print("Copy and paste this URL in your browser while your are logged in the twitter account where you want to post.") - # print("Authorization URL: " + auth_url) - - # print("Then paste the Personal Identification Number given by Twitter:") - # verifier = raw_input('PIN: ').strip() - # token = auth.get_access_token(verifier) - - # Authenticate and get the user name. - # token_key, token_secret = token - # auth.set_access_token(token_key, token_secret) - # api = tweepy.API(auth) - # username = api.me().name - # print("Authentication successful, ready to post to account: " + username) - - - # Save authentication tokens. - if not config.has_section("Info"): - config.add_section('Info') - config.set('Info','account', username) - config.set('Info','auth_date', datetime.datetime.utcnow().isoformat()) - - if not config.has_section("Auth"): - config.add_section('Auth') - config.set('Auth', 'local_token', token_key) - config.set('Auth', 'local_token_secret', token_secret) - - # Writing our configuration file to 'example.cfg' - with open(configfile, 'wb') as fd: - config.write(fd) - + def __call__(self): + logger.debug("├ call") + self.lift(self.format(self.consume())) + logger.debug("│ └OK") # # CLI # +def classes_of(namespace): + name = namespace.__name__[0].upper()+namespace.__name__[1:] + itf = getattr(namespace, name) + return [cls.__name__ for cls in itf.__subclasses__()] + + def main(): errors = {"NO_ERROR":0, "UNKNOWN_ERROR":1, "NO_SETUP_NEEDED":2, "NO_APP_KEY":10} usage = "Post text read on the standard input to a website or an API." - apis =["stdout", "twitter"] # TODO "http_post", "http_get", parser = argparse.ArgumentParser( description=usage, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + streamers = classes_of(stream) + parser.add_argument("-s", "--stream", + choices = streamers, + metavar = "STREAMER(S)", + default = ["stdin"], + action="append", + help="Where to get items.") - parser.add_argument("-a", "--api", choices=apis, default="stdout", - help="Name of the API to use.") + consumers = classes_of(consume) + parser.add_argument("-c", "--consume", + choices = consumers, + metavar = "CONSUMER", + default = "lines", + help="How to get the content form items.") - # Generic options + formaters = classes_of(format) + parser.add_argument("-f", "--format", + choices = formaters, + metavar = "FORMATER(S)", + default = "as_is", + action="append", + help="How to format items.") - parser.add_argument("-m", "--max-len", metavar="MAXLEN", type=int, default=140, - help="Maximum number of characters in the lines.") + lifters = classes_of(lift) + parser.add_argument("-l", "--lift", + choices = lifters, + metavar = "LIFTER(S)", + default = ["stdout"], + action="append", + help="How to lift items.") - parser.add_argument("-i", "--ignore", action="store_true", - help="Ignore lines that are longer than MAXLEN") - - parser.add_argument("-t", "--trim", action="store_true", - help="Trim down lines that are longer than MAXLEN.") - - parser.add_argument("-c", "--counter", action="store_true", - help="Add a counter of the form \" x/N\" at the end of the lines, \ - with N being the number of lines read and x the current index of the line. \ - NOTE: this necessitate to read all the input before processing it.") - - parser.add_argument("-q", "--quiet", action="store_true", - help="Do not print errors and warnings on the standard error output.") - - # TODO option: rate at which to post lines - - # API-dependent options - - parser.add_argument("-s", "--setup", action="store_true", - help="Setup the selected API (e.g. authenticate and get authorization to post).") - - parser.add_argument("-d", "--independent", action="store_true", - help="Post each item independently from the previous one. \ - The behaviour of this option depends on the selected API. \ - For example on Twitter: do not post a line as an answer to the previous one but as a new tweet.") - - # Twitter - parser.add_argument("--twitter-images", metavar="FILENAME(S)", nargs="+", type=str, - help="Upload each given image files along with the corresponding tweets in the sequence. If there are more images than tweets, they are silently ignored.") asked = parser.parse_args() - # Setup - if asked.setup: - configfile = asked.api+".conf" - if asked.api == "twitter": - try: - setup_twitter(configfile) - except AppKeyError as e: - if not asked.quiet: - sys.stderr.write(e.msg) - sys.exit(errors["NO_APP_KEY"]) - except: - logging.error(f"Unexpected error:\n{sys.exc_info()[0]}") - raise - else: - sys.exit(errors["NO_ERROR"]) + logging.basicConfig() + logger.setLevel("DEBUG") - else: # other API - if not asked.quiet: - sys.stderr.write("This API does not need setup.") - sys.exit(errors["NO_SETUP_NEEDED"]) + logger.debug("Available operators:") + logger.debug(f"├ streamers: {', '.join(streamers)}") + logger.debug(f"├ consumers: {', '.join(consumers)}") + logger.debug(f"├ formaters: {', '.join(formaters)}") + logger.debug(f"└ lifters: {', '.join(lifters)}") - - # Consistency checks - - if asked.ignore and asked.trim: - if not asked.quiet: - sys.stderr.write("WARNING: asking to trim AND to ignore is not logical, I will ignore.") - assert( not (asked.ignore and asked.trim) ) - - if asked.twitter_images: - if not asked.api == "twitter": - if not asked.quiet: - sys.stderr.write("WARNING: asking to upload images on twitter while not using the twitter API is not logical, I will ignore.") - assert( not (asked.twitter_images and not asked.api=="twitter") ) - - else: # Test readable images - cannot=[] - for img in asked.twitter_images: - try: - with open(img) as f: - f.read() - except OSError: - cannot.append(img) - if cannot: - logger.error("Cannot open the following image files, I will not continue: ") - for img in cannot: - logger.error(img) - sys.exit(5) # I/O Error - - # APIs - - if asked.api == "stdout": - - operate( on_stdout, asked ) - - - elif asked.api == "twitter": - - # Authenticate - config = ConfigParser.RawConfigParser() - config.read('twitter.conf') - - app_key = config.get("App","app_key") - app_secret = config.get("App","app_key_secret") - - try: - verifier_code = config.get("Auth","code") - except: - access_token = config.get("Auth","local_token") - access_token_secret = config.get("Auth","local_token_secret") - - # auth = tweepy.OAuthHandler(app_key, app_secret, "https://api.twitter.com/1.1/") - # auth.set_access_token(access_token, access_token_secret) - - # api = tweepy.API(auth) - - # # Post - # operate( on_twitter, api, asked ) + logger.debug("instantiate") + forthlift = Forthlifter() + logger.debug("run") + forthlift() + logger.debug("└OK") if __name__ == "__main__": From b8451b317150ebd5bc8c064a71d1325af3de7846 Mon Sep 17 00:00:00 2001 From: nojhan Date: Sun, 5 Apr 2026 11:38:17 +0200 Subject: [PATCH 03/13] basic features --- src/forthlift/forthlift.py | 117 ++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 6b2ebbb..fb45226 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -11,7 +11,6 @@ from configparser import ConfigParser # LIBRARY # - logger = logging.getLogger("forthlift") @@ -26,7 +25,7 @@ class stream: class consume: class Consume: - def __call__(self): + def __call__(self, stream): raise NotImplementedError class lines(Consume): @@ -35,12 +34,46 @@ class consume: class format: class Format: - def __call__(self, lines): + def __call__(self, items): raise NotImplementedError class as_is(Format): - def __call__(self, lines): - return lines + def __call__(self, items): + return items + + class trim(Format): + def __init__(self, max = 14): + self.max = max + + def trim(self, line): + if len(line) > self.max: + yield line[:self.max] + for subline in self.trim(line[self.max:]): + yield subline + else: + yield line + + def __call__(self, items): + trimmed = [] + for item in items: + for subline in self.trim(item): + trimmed.append(subline) + return trimmed + + class eol(Format): + def __call__(self, items): + eoled = [] + for item in items: + eoled.append( item + "\n" ) + return eoled + + class strip(Format): + def __call__(self, items): + stripped = [] + for item in items: + stripped.append( item.strip() ) + return stripped + class lift: class Lift: @@ -50,7 +83,10 @@ class lift: class stdout(Lift): def __call__(self, items): for item in items: - print(item, end="") + if __debug__: + print(item, end = '', file = sys.stdout, flush = True) + else: + print(item, end = '') class Forthlifter: @@ -76,22 +112,21 @@ class Forthlifter: if not isinstance(lifters, list): lifters = [lifters] self.lifters = lifters - logger.debug(f"└lifters: {', '.join(type(i).__name__ for i in self.lifters)}") + logger.debug(f"└ lifters: {', '.join(type(i).__name__ for i in self.lifters)}") def consume(self): logger.debug("│ ├ consume") - lines = [] + items = [] for streamer in self.streamers: logger.debug(f"│ │ ├ {type(self.consumer).__name__}({type(streamer).__name__})") # Concatenate - lines += self.consumer(streamer()) - logger.debug(f"│ │ ├ {len(lines)} lines") - logger.debug("│ │ └OK") - return lines + items += self.consumer(streamer()) + logger.debug(f"│ │ ├ {len(items)} items") + logger.debug(f"│ │ └OK {len(items)} items") + return items - def format(self, lines): - logger.debug(f"│ ├ format {len(lines)} lines") - items = lines + def format(self, items): + logger.debug(f"│ ├ format {len(items)} items") for formatter in self.formatters: logger.debug(f"│ │ ├ {type(formatter).__name__}") # Replace @@ -106,7 +141,7 @@ class Forthlifter: logger.debug(f"│ │ ├ {type(lifter).__name__}") # Call lifter(items) - logger.debug("│ │ └OK") + logger.debug(f"│ │ └OK {len(items)} items") def __call__(self): logger.debug("├ call") @@ -118,9 +153,10 @@ class Forthlifter: # def classes_of(namespace): - name = namespace.__name__[0].upper()+namespace.__name__[1:] - itf = getattr(namespace, name) - return [cls.__name__ for cls in itf.__subclasses__()] + itf_name = namespace.__name__[0].upper()+namespace.__name__[1:] + itf = getattr(namespace, itf_name) + subs = {cls.__name__:cls for cls in itf.__subclasses__()} + return subs def main(): @@ -130,34 +166,35 @@ def main(): parser = argparse.ArgumentParser( description=usage, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + streamers = classes_of(stream) parser.add_argument("-s", "--stream", - choices = streamers, - metavar = "STREAMER(S)", - default = ["stdin"], + choices = streamers.keys(), + # metavar = "STREAMER(S)", + default = [], action="append", help="Where to get items.") consumers = classes_of(consume) parser.add_argument("-c", "--consume", - choices = consumers, - metavar = "CONSUMER", + choices = consumers.keys(), + # metavar = "CONSUMER", default = "lines", help="How to get the content form items.") formaters = classes_of(format) parser.add_argument("-f", "--format", - choices = formaters, - metavar = "FORMATER(S)", - default = "as_is", + choices = formaters.keys(), + # metavar = "FORMATER(S)", + default = [], action="append", help="How to format items.") lifters = classes_of(lift) parser.add_argument("-l", "--lift", - choices = lifters, - metavar = "LIFTER(S)", - default = ["stdout"], + choices = lifters.keys(), + # metavar = "LIFTER(S)", + default = [], action="append", help="How to lift items.") @@ -173,9 +210,23 @@ def main(): logger.debug(f"├ formaters: {', '.join(formaters)}") logger.debug(f"└ lifters: {', '.join(lifters)}") - logger.debug("instantiate") - forthlift = Forthlifter() - logger.debug("run") + if not asked.stream: + asked.stream = ["stdin"] + + if not asked.format: + asked.format = ["as_is"] + + if not asked.lift: + asked.lift = ["stdout"] + + logger.debug("Chosen operators:") + forthlift = Forthlifter( + consumer = consumers[asked.consume](), + streamers = [streamers[k]() for k in asked.stream], + formatters = [formaters[k]() for k in asked.format], + lifters = [ lifters[k]() for k in asked.lift ], + ) + logger.debug("Run:") forthlift() logger.debug("└OK") From 20d0424930c8cd5a731fc8b8a95ebe5020220b12 Mon Sep 17 00:00:00 2001 From: nojhan Date: Sun, 5 Apr 2026 11:57:14 +0200 Subject: [PATCH 04/13] feat: operator arguments --- src/forthlift/forthlift.py | 46 +++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index fb45226..8592dcd 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import re import sys import locale import logging @@ -42,8 +43,8 @@ class format: return items class trim(Format): - def __init__(self, max = 14): - self.max = max + def __init__(self, max = "14"): + self.max = int(max) def trim(self, line): if len(line) > self.max: @@ -158,6 +159,23 @@ def classes_of(namespace): subs = {cls.__name__:cls for cls in itf.__subclasses__()} return subs +def operator(asked_op): + logger.debug(f"├ Parsed operators:") + for op in asked_op: + # We do not use a f(a) notation, + # because the shell would try to interpret + # any unescaped/unquoted parentheses. + # It is faster to use f:a for the user. + m = re.match("(\w+):(.+)", op) + if m: + name = m.group(1) + args = [a.strip() for a in m.group(2).split(',')] + else: + name = op + args = [] + logger.debug(f"│ ├ {name}({','.join(args)})") + yield name,args + logger.debug("│ └OK") def main(): errors = {"NO_ERROR":0, "UNKNOWN_ERROR":1, "NO_SETUP_NEEDED":2, "NO_APP_KEY":10} @@ -169,31 +187,31 @@ def main(): streamers = classes_of(stream) parser.add_argument("-s", "--stream", - choices = streamers.keys(), - # metavar = "STREAMER(S)", + # choices = streamers.keys(), + metavar = '{' + ','.join(streamers) + '}', default = [], action="append", help="Where to get items.") consumers = classes_of(consume) parser.add_argument("-c", "--consume", - choices = consumers.keys(), - # metavar = "CONSUMER", + # choices = consumers.keys(), + metavar = '{' + ','.join(consumers) + '}', default = "lines", help="How to get the content form items.") formaters = classes_of(format) parser.add_argument("-f", "--format", - choices = formaters.keys(), - # metavar = "FORMATER(S)", + # choices = formaters.keys(), + metavar = '{' + ','.join(formaters) + '}', default = [], action="append", help="How to format items.") lifters = classes_of(lift) parser.add_argument("-l", "--lift", - choices = lifters.keys(), - # metavar = "LIFTER(S)", + # choices = lifters.keys(), + metavar = '{' + ','.join(lifters) + '}', default = [], action="append", help="How to lift items.") @@ -210,6 +228,8 @@ def main(): logger.debug(f"├ formaters: {', '.join(formaters)}") logger.debug(f"└ lifters: {', '.join(lifters)}") + # Sane defaults. Cannot be a default in argparse, + # because choices would be append to it. if not asked.stream: asked.stream = ["stdin"] @@ -222,9 +242,9 @@ def main(): logger.debug("Chosen operators:") forthlift = Forthlifter( consumer = consumers[asked.consume](), - streamers = [streamers[k]() for k in asked.stream], - formatters = [formaters[k]() for k in asked.format], - lifters = [ lifters[k]() for k in asked.lift ], + streamers = [streamers[op](*args) for op,args in operator(asked.stream)], + formatters = [formaters[op](*args) for op,args in operator(asked.format)], + lifters = [ lifters[op](*args) for op,args in operator(asked.lift )], ) logger.debug("Run:") forthlift() From d35d0df021a3db0cc7bf9d0236f7b4fc30760300 Mon Sep 17 00:00:00 2001 From: nojhan Date: Sun, 5 Apr 2026 17:02:26 +0200 Subject: [PATCH 05/13] feat: more operators --- pyproject.toml | 1 + src/forthlift/forthlift.py | 109 +++++++++++++++++++++++++++++++------ 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5e90228..d7af3be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "argparse>=1.4.0", "configparser>=7.2.0", "datetime>=6.0", + "rich>=14.3.3", ] [project.scripts] diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 8592dcd..31bb264 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -6,7 +6,10 @@ import locale import logging import argparse import datetime + from configparser import ConfigParser +from rich.panel import Panel +from rich import print # # LIBRARY @@ -33,6 +36,19 @@ class consume: def __call__(self, stream): return stream.readlines() + class paragraphs(Consume): + def __call__(self, stream): + pars = [] + current = "" + for item in stream.readlines(): + # Not counting spaces as legit content. + if item.strip(): + current += item + else: + pars.append( current ) + current = "" + return pars + class format: class Format: def __call__(self, items): @@ -43,22 +59,23 @@ class format: return items class trim(Format): - def __init__(self, max = "14"): + # Every argument is a string. + def __init__(self, max = "140"): self.max = int(max) - def trim(self, line): - if len(line) > self.max: - yield line[:self.max] - for subline in self.trim(line[self.max:]): - yield subline + def trim(self, item): + if len(item) > self.max: + yield item[:self.max] + for subitem in self.trim(item[self.max:]): + yield subitem else: - yield line + yield item def __call__(self, items): trimmed = [] for item in items: - for subline in self.trim(item): - trimmed.append(subline) + for separated in self.trim(item): + trimmed.append(separated) return trimmed class eol(Format): @@ -75,6 +92,63 @@ class format: stripped.append( item.strip() ) return stripped + class skip(Format): + def __call__(self, items): + res = [] + for item in items: + if item.strip(): + res.append( item ) + return res + + class paragraph(Format): + def __call__(self, items): + glued = [] + current = "" + for item in items: + if item.strip(): + current += item + else: + glued.append( current ) + current = "" + return glued + + class panel(Format): + def __call__(self, items): + panel = [] + for item in items: + panel.append( Panel.fit(item.strip()) ) + return panel + + class count(Format): + def __init__(self, end = ' ␄', sep = '\n'): + self.sep = sep + self.end = end + + def __call__(self, items): + total = len(items) + res = [] + for i,item in enumerate(items): + res.append(f"{item}{self.sep}{i+1}/{total}") + if res: + res[-1] += self.end + return res + + class suffix(Format): + def __init__(self, content = "", sep = '\n'): + self.content = content + self.sep = sep + + def __call__(self, items): + return [i+self.sep+self.content for i in items] + + class prefix(Format): + def __init__(self, content = "", sep = '\n'): + self.content = content + self.sep = sep + + def __call__(self, items): + return [self.content+self.sep+i for i in items] + class lift: class Lift: @@ -84,10 +158,13 @@ class lift: class stdout(Lift): def __call__(self, items): for item in items: - if __debug__: - print(item, end = '', file = sys.stdout, flush = True) + if item: + if __debug__: + print(item, end = '', file = sys.stdout, flush = True) + else: + print(item, end = '') else: - print(item, end = '') + logger.debug("Empty item") class Forthlifter: @@ -97,14 +174,14 @@ class Forthlifter: formatters = [format.as_is()], lifters = [lift.stdout()], ): - self.consumer = consumer - logger.debug(f"├ consumer: {type(consumer).__name__}") - if not isinstance(streamers, list): streamers = [streamers] self.streamers = streamers logger.debug(f"├ streamers: {', '.join(type(i).__name__ for i in self.streamers)}") + self.consumer = consumer + logger.debug(f"├ consumer: {type(consumer).__name__}") + if not isinstance(formatters, list): formatters = [formatters] self.formatters = formatters @@ -166,7 +243,7 @@ def operator(asked_op): # because the shell would try to interpret # any unescaped/unquoted parentheses. # It is faster to use f:a for the user. - m = re.match("(\w+):(.+)", op) + m = re.match(r"(\w+):(.+)", op) if m: name = m.group(1) args = [a.strip() for a in m.group(2).split(',')] From 70661d39b8f4d74673bb9a4e568f727ede4f0173 Mon Sep 17 00:00:00 2001 From: nojhan Date: Sun, 5 Apr 2026 19:00:18 +0200 Subject: [PATCH 06/13] feat: adds consume.sections --- src/forthlift/forthlift.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 31bb264..3f22fbd 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -49,6 +49,26 @@ class consume: current = "" return pars + class sections(Consume): + def __init__(self, mark = r"^#", skip = "False"): + self.mark = mark + self.skip = bool(skip) + + def __call__(self, stream): + sec = [] + current = "" + for item in stream.readlines(): + if re.match(self.mark, item[0]): + sec.append( current ) + if self.skip: + current = "" + else: + current = item + else: + current += item + sec.append( current ) + return sec + class format: class Format: def __call__(self, items): @@ -317,8 +337,11 @@ def main(): asked.lift = ["stdout"] logger.debug("Chosen operators:") + + cop,cargs = list(operator([asked.consume]))[0] + forthlift = Forthlifter( - consumer = consumers[asked.consume](), + consumer = consumers[cop](*cargs), streamers = [streamers[op](*args) for op,args in operator(asked.stream)], formatters = [formaters[op](*args) for op,args in operator(asked.format)], lifters = [ lifters[op](*args) for op,args in operator(asked.lift )], From 77bd035f723f34e44933e56370d15bbe6bc81ab3 Mon Sep 17 00:00:00 2001 From: nojhan Date: Sun, 5 Apr 2026 19:10:36 +0200 Subject: [PATCH 07/13] feat: adds consume.nlines --- src/forthlift/forthlift.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 3f22fbd..892fb68 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -69,6 +69,27 @@ class consume: sec.append( current ) return sec + class nlines(Consume): + def __init__(self, nb = "10"): + self.nb = int(nb) + + def __call__(self, stream): + sec = [] + count = 0 + current = "" + for item in stream.readlines(): + if count >= self.nb: + sec.append( current ) + current = "" + count = 0 + else: + current += item + count += 1 + sec.append( current ) + return sec + + + class format: class Format: def __call__(self, items): From e0adccda32b0f7a9032993619e32b8fde97a6730 Mon Sep 17 00:00:00 2001 From: nojhan Date: Mon, 6 Apr 2026 14:12:01 +0200 Subject: [PATCH 08/13] feat(doc): autoextract operators' help --- src/forthlift/forthlift.py | 134 +++++++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 35 deletions(-) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 892fb68..6401913 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -4,6 +4,7 @@ import re import sys import locale import logging +import inspect import argparse import datetime @@ -24,6 +25,7 @@ class stream: raise NotImplementedError class stdin(Stream): + """Stream from the standard input.""" def __call__(self): return sys.stdin @@ -33,10 +35,12 @@ class consume: raise NotImplementedError class lines(Consume): + """Consume line by line.""" def __call__(self, stream): return stream.readlines() class paragraphs(Consume): + """Consume paragraph by paragraph (separated by an empty line).""" def __call__(self, stream): pars = [] current = "" @@ -50,9 +54,15 @@ class consume: return pars class sections(Consume): - def __init__(self, mark = r"^#", skip = "False"): + """Consume section by section. A new section starts when a line matches the `mark` regexp. If `skip` is set to 'skip', the marked line is not consumed.""" + def __init__(self, mark = r"^#", skip = "noskip"): self.mark = mark - self.skip = bool(skip) + if skip == 'skip': + self.skip = True + elif skip == 'noskip': + self.skip = False + else: + self.skip = bool(skip) def __call__(self, stream): sec = [] @@ -70,6 +80,7 @@ class consume: return sec class nlines(Consume): + """Consume by groups of `nb` lines.""" def __init__(self, nb = "10"): self.nb = int(nb) @@ -89,17 +100,18 @@ class consume: return sec - class format: class Format: def __call__(self, items): raise NotImplementedError - class as_is(Format): + class asis(Format): + """Do not format anything.""" def __call__(self, items): return items class trim(Format): + """Split items if their length is longer than `max`, and create new items with the remaining parts.""" # Every argument is a string. def __init__(self, max = "140"): self.max = int(max) @@ -120,6 +132,7 @@ class format: return trimmed class eol(Format): + """Add an end of line after the item.""" def __call__(self, items): eoled = [] for item in items: @@ -127,6 +140,7 @@ class format: return eoled class strip(Format): + """Remove any space character around the item.""" def __call__(self, items): stripped = [] for item in items: @@ -134,6 +148,7 @@ class format: return stripped class skip(Format): + """Skip items containing only spaces or being empty.""" def __call__(self, items): res = [] for item in items: @@ -141,7 +156,8 @@ class format: res.append( item ) return res - class paragraph(Format): + class glue(Format): + """Glue consecutive items together if they are not separated by an empty one.""" def __call__(self, items): glued = [] current = "" @@ -154,13 +170,15 @@ class format: return glued class panel(Format): + """Surround each item by an ascii-art box. NOTE: only works when lifted on stdout.""" def __call__(self, items): panel = [] for item in items: panel.append( Panel.fit(item.strip()) ) return panel - class count(Format): + class counter(Format): + """Add a counter at the end of each items, with the current index and the total. If `end` is given, it is added at the very last item. If `sep` is given, it is appended to the item before the counter itself.""" def __init__(self, end = ' ␄', sep = '\n'): self.sep = sep self.end = end @@ -175,6 +193,7 @@ class format: return res class suffix(Format): + """Add the `content` string after each item. If `sep` is given, it is appended to the item before the content and after the item.""" def __init__(self, content = "", sep = '\n'): self.content = content self.sep = sep @@ -183,6 +202,7 @@ class format: return [i+self.sep+self.content for i in items] class prefix(Format): + """Add the `content` string before each item. If `sep` is given, it is prepended to the item after the content and before the item.""" def __init__(self, content = "", sep = '\n'): self.content = content self.sep = sep @@ -197,6 +217,7 @@ class lift: raise NotImplementedError class stdout(Lift): + """Print the items on the standard output.""" def __call__(self, items): for item in items: if item: @@ -212,7 +233,7 @@ class Forthlifter: def __init__(self, consumer = consume.lines(), streamers = [stream.stdin()], - formatters = [format.as_is()], + formatters = [format.asis()], lifters = [lift.stdout()], ): if not isinstance(streamers, list): @@ -277,6 +298,7 @@ def classes_of(namespace): subs = {cls.__name__:cls for cls in itf.__subclasses__()} return subs + def operator(asked_op): logger.debug(f"├ Parsed operators:") for op in asked_op: @@ -295,44 +317,86 @@ def operator(asked_op): yield name,args logger.debug("│ └OK") + +def help_op(ops): + h = "" + itf = list(ops.values())[0].__mro__[1].__name__.lower() + h += f"\nAVAILABLE OPERATORS FOR --{itf}:\n" + for name,cls in ops.items(): + # Signature + hsig = f" -{itf[0]} {name.lower()}" + args = inspect.getfullargspec(cls)[0] + if args: + sign = inspect.signature(cls) + args = [sign.parameters[a].name for a in sign.parameters] + hsig += f"[:{','.join(args)}]" + h += hsig + + # Example (using defaults) + hex = f"-{itf[0]} " + args = inspect.getfullargspec(cls)[0] + if args: + sep = ":" + sign = inspect.signature(cls) + defs = [sign.parameters[a].default for a in sign.parameters] + if not all(defs) or any(re.match(r'\s', d) for d in defs): + hdefs = [d.replace("\n", r"\n") for d in defs] + hex += f"'{name.lower()}{sep}{','.join(hdefs)}'" + else: + hex += f"{name.lower()}{sep}{','.join(defs)}" + h += f"\n\tDefault: {hex}" + else: + hex += name.lower() + + if cls.__doc__: + h+= f"\n\t{cls.__doc__}\n" + else: + h += "\n" + return h + def main(): errors = {"NO_ERROR":0, "UNKNOWN_ERROR":1, "NO_SETUP_NEEDED":2, "NO_APP_KEY":10} usage = "Post text read on the standard input to a website or an API." - parser = argparse.ArgumentParser( description=usage, - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - streamers = classes_of(stream) - parser.add_argument("-s", "--stream", - # choices = streamers.keys(), - metavar = '{' + ','.join(streamers) + '}', - default = [], - action="append", - help="Where to get items.") - consumers = classes_of(consume) - parser.add_argument("-c", "--consume", - # choices = consumers.keys(), - metavar = '{' + ','.join(consumers) + '}', - default = "lines", - help="How to get the content form items.") - formaters = classes_of(format) - parser.add_argument("-f", "--format", - # choices = formaters.keys(), - metavar = '{' + ','.join(formaters) + '}', - default = [], - action="append", - help="How to format items.") - lifters = classes_of(lift) - parser.add_argument("-l", "--lift", - # choices = lifters.keys(), - metavar = '{' + ','.join(lifters) + '}', + + epilog = "" + epilog += help_op(streamers) + epilog += help_op(consumers) + epilog += help_op(formaters) + epilog += help_op(lifters) + + parser = argparse.ArgumentParser( + description=usage, + formatter_class = argparse.RawTextHelpFormatter, + epilog = epilog) + + parser.add_argument("-s", "--stream", + metavar = "STREAM(S)", default = [], action="append", - help="How to lift items.") + help="Where to get items (several occurences possibles, order matters).") + + parser.add_argument("-c", "--consume", + metavar = "CONSUME", + default = "lines", + help="How to extract the content from the stream (only one occurence).") + + parser.add_argument("-f", "--format", + metavar = "FORMAT(S)", + default = [], + action="append", + help="How to format items (several occurences possibles, order matters).") + + parser.add_argument("-l", "--lift", + metavar = "LIFT(S)", + default = [], + action="append", + help="How to send items somewhere (several occurences possibles, order matters).") asked = parser.parse_args() @@ -352,7 +416,7 @@ def main(): asked.stream = ["stdin"] if not asked.format: - asked.format = ["as_is"] + asked.format = ["asis"] if not asked.lift: asked.lift = ["stdout"] From 8179e121f5818ab6cfb30c787eea2a98187ffb8e Mon Sep 17 00:00:00 2001 From: nojhan Date: Mon, 6 Apr 2026 16:09:49 +0200 Subject: [PATCH 09/13] feat(stream): adds filename --- src/forthlift/forthlift.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 6401913..8611958 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -29,6 +29,18 @@ class stream: def __call__(self): return sys.stdin + class filename(Stream): + """Stream from the given file.""" + def __init__(self, filename = ""): + self.filename = filename + + def __call__(self): + self.fd = open(self.filename, 'r') + return self.fd + + def __del__(self): + self.fd.close() + class consume: class Consume: def __call__(self, stream): From 1418055706841929a477ff877f96a652aadf0e08 Mon Sep 17 00:00:00 2001 From: nojhan Date: Mon, 6 Apr 2026 16:10:05 +0200 Subject: [PATCH 10/13] refactor: make all operators generators Except `format.counter`, of course. --- src/forthlift/forthlift.py | 104 ++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 54 deletions(-) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 8611958..c358c38 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -49,21 +49,22 @@ class consume: class lines(Consume): """Consume line by line.""" def __call__(self, stream): - return stream.readlines() + for line in stream: + yield line class paragraphs(Consume): """Consume paragraph by paragraph (separated by an empty line).""" def __call__(self, stream): - pars = [] current = "" - for item in stream.readlines(): + for item in stream: # Not counting spaces as legit content. if item.strip(): current += item else: - pars.append( current ) + yield current current = "" - return pars + if current.strip(): + yield current class sections(Consume): """Consume section by section. A new section starts when a line matches the `mark` regexp. If `skip` is set to 'skip', the marked line is not consumed.""" @@ -77,19 +78,17 @@ class consume: self.skip = bool(skip) def __call__(self, stream): - sec = [] current = "" - for item in stream.readlines(): + for item in stream: if re.match(self.mark, item[0]): - sec.append( current ) + yield current if self.skip: current = "" else: current = item else: current += item - sec.append( current ) - return sec + yield current class nlines(Consume): """Consume by groups of `nb` lines.""" @@ -97,19 +96,17 @@ class consume: self.nb = int(nb) def __call__(self, stream): - sec = [] count = 0 current = "" - for item in stream.readlines(): + for item in stream: if count >= self.nb: - sec.append( current ) + yield current current = "" count = 0 else: current += item count += 1 - sec.append( current ) - return sec + yield current class format: @@ -120,7 +117,8 @@ class format: class asis(Format): """Do not format anything.""" def __call__(self, items): - return items + for item in items: + yield item class trim(Format): """Split items if their length is longer than `max`, and create new items with the remaining parts.""" @@ -137,57 +135,47 @@ class format: yield item def __call__(self, items): - trimmed = [] for item in items: for separated in self.trim(item): - trimmed.append(separated) - return trimmed + yield separated class eol(Format): """Add an end of line after the item.""" def __call__(self, items): - eoled = [] for item in items: - eoled.append( item + "\n" ) - return eoled + yield item + "\n" class strip(Format): """Remove any space character around the item.""" def __call__(self, items): - stripped = [] for item in items: - stripped.append( item.strip() ) - return stripped + yield item.strip() class skip(Format): """Skip items containing only spaces or being empty.""" def __call__(self, items): - res = [] for item in items: if item.strip(): - res.append( item ) - return res + yield item class glue(Format): """Glue consecutive items together if they are not separated by an empty one.""" def __call__(self, items): - glued = [] current = "" for item in items: if item.strip(): current += item else: - glued.append( current ) + yield current current = "" - return glued + if current.strip(): + yield current class panel(Format): """Surround each item by an ascii-art box. NOTE: only works when lifted on stdout.""" def __call__(self, items): - panel = [] for item in items: - panel.append( Panel.fit(item.strip()) ) - return panel + yield Panel.fit(item.strip()) class counter(Format): """Add a counter at the end of each items, with the current index and the total. If `end` is given, it is added at the very last item. If `sep` is given, it is appended to the item before the counter itself.""" @@ -195,14 +183,16 @@ class format: self.sep = sep self.end = end - def __call__(self, items): + def __call__(self, counted): + items = list(counted) # Consume everything at once. total = len(items) res = [] for i,item in enumerate(items): res.append(f"{item}{self.sep}{i+1}/{total}") if res: res[-1] += self.end - return res + for r in res: + yield r class suffix(Format): """Add the `content` string after each item. If `sep` is given, it is appended to the item before the content and after the item.""" @@ -211,7 +201,8 @@ class format: self.sep = sep def __call__(self, items): - return [i+self.sep+self.content for i in items] + for i in items: + yield i+self.sep+self.content class prefix(Format): """Add the `content` string before each item. If `sep` is given, it is prepended to the item after the content and before the item.""" @@ -220,25 +211,33 @@ class format: self.sep = sep def __call__(self, items): - return [self.content+self.sep+i for i in items] + for i in items: + yield self.content+self.sep+i class lift: class Lift: - def __call__(self, items): + def call(self, items): raise NotImplementedError + def __call__(self, items): + count = 0 + for item in items: + self.call(item) + count += 1 + logger.debug(f"│ │ ├ {count}th item") + logger.debug(f"│ │ └OK {count} items") + class stdout(Lift): """Print the items on the standard output.""" - def __call__(self, items): - for item in items: - if item: - if __debug__: - print(item, end = '', file = sys.stdout, flush = True) - else: - print(item, end = '') + def call(self, item): + if item: + if __debug__: + print(item, end = '', file = sys.stdout, flush = True) else: - logger.debug("Empty item") + print(item, end = '') + else: + logger.debug("Empty item") class Forthlifter: @@ -273,27 +272,24 @@ class Forthlifter: logger.debug(f"│ │ ├ {type(self.consumer).__name__}({type(streamer).__name__})") # Concatenate items += self.consumer(streamer()) - logger.debug(f"│ │ ├ {len(items)} items") - logger.debug(f"│ │ └OK {len(items)} items") + logger.debug(f"│ │ └OK") return items def format(self, items): - logger.debug(f"│ ├ format {len(items)} items") + logger.debug(f"│ ├ format") for formatter in self.formatters: logger.debug(f"│ │ ├ {type(formatter).__name__}") # Replace items = formatter(items) - logger.debug(f"│ │ ├ {len(items)} items") - logger.debug(f"│ │ └OK {len(items)} items") + logger.debug(f"│ │ └OK") return items def lift(self, items): - logger.debug(f"│ ├ lift {len(items)} items") + logger.debug(f"│ ├ lift") for lifter in self.lifters: logger.debug(f"│ │ ├ {type(lifter).__name__}") # Call lifter(items) - logger.debug(f"│ │ └OK {len(items)} items") def __call__(self): logger.debug("├ call") From b50f5fa79c12c019ef17f2dcdd3b4e51911ffcee Mon Sep 17 00:00:00 2001 From: nojhan Date: Sat, 11 Apr 2026 17:11:01 +0200 Subject: [PATCH 11/13] update README --- README.md | 166 ++++++++++++++++++------------------- src/forthlift/forthlift.py | 2 +- 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 6b7e022..2ebdfd6 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,102 @@ forthlift — post sequences of texts on social media =================================================== -Forthlift is a command line application to post sequences of text lines on social media. +Forthlift's primary use case is a command line application to post sequences of +text items on social media. You first prepare your thread in your text editor, +then call forthlift, and everything is posted at once. -Its main use case is to post on Twitter several status that you have prepared first in your text editor. +As side effect, fothlift became a more generic tool, that can process text +streams, format them, assemble their atomic units in anothers, and then send +them elsewhere. It is thus designed to ease automation and integration with +existing tools, and has an Unix-like text-and-pipes philosophy. -It is thus designed to ease automation and integration in existing tools -and had an Unix-like text-and-pipes command line interface. +## Examples -For example, it makes it easy to post "chained" twitter status (that answer to each others) -from your text editor. Just send the selected text to its standard input, selecting the -twitter API, and voilà. Using Vim, you can tweet the visually-selected lines with: -`:'<,'>w !forthlift -a twitter` +### Select in Vim, post on Mastodon + +Forthlift makes it easy to post "chained" Mastodon toots +(that answer to each others) from your text editor. +Just send the selected text to its standard input, selecting the +twitter API, and voilà. + +Using Vim, you can toot the visually-selected lines (each line making a toot) +with: +```vim +:'<,'>w !forthlift --lift mastodon +``` + +### Prepare toots in a file, post on Mastodon + +Let's say that you prepared a text file of the form: + +``` +This is a multi-lines... + +... toot! +-- +And here is an answeer to the previous toot. +(With two lines as well.) +-- +And a final toot. +``` + +You would want the toots to show a counter, managing the expectations of your +reader. + +You can post it right away with: +```sh +cat my_file.txt | forthlift --consume "sections:--,skip" --format counter -l mastodon +``` + +### Manage long threads from Markdown + +Let's say you want to post the sections of this README as a sequence of toots, +with a hashtag that indicates it's gona be long. +And you would want to double-check what it would do first. You can send the +items on the standard output, like: + +```sh +forthlift -s filename:README.md -c sections -f suffix:#longThread -f panel -l stdout +``` -## SYNOPSIS +### Full-featured file-to-Mastodon pipeline -`forthlift` [-h] - -`forthlift` [-a {stdout,twitter}] [-m MAXLEN] [-i|-t] [-c] [-q] [-d] [-s] +```sh +# Define a shortcut +mastopost() { forthlift -s "filename:$1" -c sections:^#,skip -f skip -f counter -f suffix:#longThread -l mastodon; } +# Call it with: +mastopost my_file.txt +``` ## DESCRIPTION ### A generic tool -Generally speaking, it's a Unix-like command that operate a sequence of pre-programmed -chained actions on its text input. +Generally speaking, Forthlift is a Unix-like command that operate a sequence of +pre-programmed chained actions on its text input. -It comes with some existing actions: -* `stdout`: print the input text on the standard output, -* `twitter`: send the input text as status on twitter. +Forthlift has four action operators classes, one for each step of the processing: + +1. `stream`, indicating from where to get the input data. +2. `consume`, telling *how* to parse the data from the input stream. +3. `format`, for manipulating the content itself. +4. `lift`, defining where to send the final items. + +Only one `consume` operator can be passed, but any number of `stream`, `format`, +and `lift` operators can be combined. +To apply several operators of the same class, the user must pass the same flag +several time. For instance: + +```sh +# Applies strip, then skip, then counter on the consumed lines. +forthlift --format strip --format skip --format counter +``` + +Most of the time, the order of the operators matters. -### Features - -The main feature of forthlift is its ability to *chain* actions. -Depending on the chosen API, this could means different things: - -* for twitter, this mean that the sequence of status will be posted - as a sequence of *answers* and not as a list of independent tweets. - -* for the "stdout" API, this means that each printed line will start - with its index in the input list. - - -While it is recommended to prepare the input text with other text-processing tools -(fold, fmt, tr, sed, grep, your text editor, etc.), -forthlift comes with some rough text-processing capabilities, among which: - -* ignore or trim lines that are longer than a given size (see `--trim` and `--ignore`), - -* add a counter of the form `/` at the end of the lines (see `--counter`). - - -## OPTIONS - -* -h, --help: show this help message and exit - -* -a {stdout,twitter}, --api {stdout,twitter}: Name of the API to use. - (default: stdout) - -* -m MAXLEN, --max-len MAXLEN: Maximum number of characters in the lines. - (default: 140) - -* -i, --ignore: Ignore lines that are longer than MAXLEN (default: False) - -* -t, --trim: Trim down lines that are longer than MAXLEN. (default: False) - -* -c, --counter: Add a counter of the form " x/N" at the end of the lines, - with N being the number of lines read and x the current index of the line. - NOTE: this necessitate to read all the input before processing it. - (default: False) - -* -q, --quiet: Do not print errors and warnings on the standard error output. - (default: False) - -* -s, --setup: Setup the selected API (e.g. authenticate and get authorization to post). - (default: False) - -* -d, --independent: Post each item independently from the previous one - The behaviour of this option depends on the selected API. - For example on Twitter: do not post a line as an answer to the previous one but as a new tweet. - (default: False) - -* --twitter-images FILENAME(S) [FILENAME(S) ...]: - Upload each given image files along with the corresponding tweets in the sequence. - If there are more images than tweets, they are silently ignored. - (default: None) - - -## INSTALLATION - -### Twitter - -1) Copy `twitter.conf-dist` as `twitter.conf` and indicate your developer's API keys in the corresponding fields. -2) Run `forthlift --api twitter --setup` and follow the instructions (go to the given URL, then paste the given PIN). - -Why should you get developer's API keys? Because Twitter does not like open-source desktop applications, see: -http://arstechnica.com/security/2010/09/twitter-a-case-study-on-how-to-do-oauth-wrong/ +## SYNOPSIS +{{FORTHLIFT_HELP}} diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index c358c38..2aeccbb 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -159,7 +159,7 @@ class format: yield item class glue(Format): - """Glue consecutive items together if they are not separated by an empty one.""" + """Glue consecutive items together if they are not separated by an empty one. Make a new item after an empty one.""" def __call__(self, items): current = "" for item in items: From bb00eabd70449ccb38b0903d536e36a50dbce9ad Mon Sep 17 00:00:00 2001 From: nojhan Date: Sun, 12 Apr 2026 22:33:43 +0200 Subject: [PATCH 12/13] feat(lift): mastodon PoC --- pyproject.toml | 2 + src/forthlift/forthlift.py | 201 ++++++++++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d7af3be..4776791 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,9 @@ dependencies = [ "argparse>=1.4.0", "configparser>=7.2.0", "datetime>=6.0", + "mastodon-py>=2.1.4", "rich>=14.3.3", + "toml>=0.10.2", ] [project.scripts] diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index 2aeccbb..fe03c55 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -1,12 +1,19 @@ #!/usr/bin/env python +import io +import os import re import sys +import toml +import time import locale import logging import inspect +import tempfile import argparse import datetime +import mastodon +import webbrowser from configparser import ConfigParser from rich.panel import Panel @@ -217,10 +224,12 @@ class format: class lift: class Lift: - def call(self, items): + def call(self, item): + """Interface for posting a single independant item.""" raise NotImplementedError def __call__(self, items): + """Interface for posting all items.""" count = 0 for item in items: self.call(item) @@ -240,6 +249,171 @@ class lift: logger.debug("Empty item") + class mastodon(Lift): + + def __init__(self, dryrun = 'dry'): + self._scopes = ['read', 'write'] + + if dryrun == 'dry': + self.dry_run = True + elif dryrun == 'nodry': + self.dry_run = False + else: + msg = f"I do not understand what you mean by `{dryrun}`. Please either indicate `dry` or `nodry`." + logger.error(msg) + raise RuntimeError(msg) + + pyp = toml.load("pyproject.toml") + self.name = pyp["project"]["name"] + self.version = pyp["project"]["version"] + + self.config_path = f"{self.name}.toml" + if not os.path.isfile(self.config_path): + with open(self.config_path, 'w') as fd: + fd.write("") + self.config = self.load_config() + + if self.needs_init(): + self.init() + + self.masto = mastodon.Mastodon( + api_base_url = self.config["instance"], + # client_id = self.config["client_id"], + client_secret = self.config["client_secret"], + access_token = self.config["token"], + user_agent = f"{self.name}:{self.version}" + ) + + logger.debug(self.config) + + def needs_init(self): + if not self.config["instance"] \ + or not self.config["account"] \ + or not self.config["client_id"] \ + or not self.config["client_secret"] \ + or not self.config["token"]: + return True + else: + return False + + def init(self): + if not self.needs_init(): + print(f"The {__name__} operator has already been initialized") + print("Current configuration is:") + print("\tinstance:", self.config["instance"]) + print("\tuser:", self.config["account"]) + + ans = input("Do you want to re-init it? (yes/no): ").strip() + if ans[0].lower() == 'n': + return + elif ans[1].lower() != 'y': + print("I did not understand your answer, " + " I'll assume you meant `no` and stop here.") + return + + print("URL of your Mastodon instance?") + print("For example: https://social.antigene.org") + # instance = input("URL: ").strip() + instance = "https://social.antigene.org" + self.config["instance"] = instance + + self.register_app() + self.oauth() + + def save_config(self, config): + logger.debug(f"Save config in: {self.config_path}") + with open(self.config_path, 'w') as fd: + toml.dump(config, fd) + + def load_config(self): + logger.debug(f"Load config from: {self.config_path}") + config = { + "instance": None, + "account": None, + "client_id": None, + "client_secret": None, + "token": None + } + local_config = toml.load(self.config_path) + config.update(local_config) + logger.debug(config) + return config + + def register_app(self): + logger.debug(f"Register {self.name} on {self.config['instance']}") + + client_id, client_secret = mastodon.Mastodon.create_app( + self.name, + scopes = self._scopes, + api_base_url = self.config["instance"], + website = "https://nojhan.net/git/nojhan/forthlift", + user_agent = f"{self.name}:{self.version}", + ) + + self.config["client_id"] = client_id + logger.debug(f"ID: {client_id}") + self.config["client_secret"] = client_secret + logger.debug(f"Secret: {client_secret}") + self.save_config(self.config) + + def oauth(self): + logger.debug(f"OAuth to: {self.config['instance']}") + oauth = mastodon.Mastodon( + client_id = self.config["client_id"], + client_secret = self.config["client_secret"], + api_base_url = self.config["instance"], + ) + oauth_url = oauth.auth_request_url(scopes = self._scopes) + logger.debug(f"Opening web page: {oauth_url}") + logger.debug("Please log in there.") + webbrowser.open_new(oauth_url) + time.sleep(5) + print("After logging in, paste here the code you received:") + oauth_code = input("Code: ").strip() + + token = oauth.log_in( + code = oauth_code, + scopes = self._scopes, + ) + self.config["token"] = token + logger.debug(f"Token: {token}") + + account = oauth.me().acct + self.config["account"] = account + logger.debug(f"Account: {account}") + self.save_config(self.config) + + def post(self, item, prev_status = None): + if self.dry_run: + print(item) + else: + print(item) + + def __call__(self, items): + n = 0 + first_item = next(items, None) + if first_item == None: + logger.error("No item to post") + return + else: + logger.debug(f"Post {n+1}") + n += 1 + # prev_status = self.masto.status_post( + # status = first_item, + # ) + # logger.debug(prev_status) + self.post(first_item) + for item in items: + logger.debug(f"Post {n+1}") + n += 1 + # prev_status = self.masto.status_reply( + # to_status = prev_status, + # status = item, + # ) + # logger.debug(prev_status) + self.post(item) + logger.debug(f"Posted {n} items") + class Forthlifter: def __init__(self, consumer = consume.lines(), @@ -367,11 +541,13 @@ def main(): usage = "Post text read on the standard input to a website or an API." + # Dictionaries of {name: class} streamers = classes_of(stream) consumers = classes_of(consume) formaters = classes_of(format) lifters = classes_of(lift) + # Extract docstrings epilog = "" epilog += help_op(streamers) epilog += help_op(consumers) @@ -406,12 +582,35 @@ def main(): action="append", help="How to send items somewhere (several occurences possibles, order matters).") + initializables = [] + + for cls in \ + list(streamers.values()) \ + + list(consumers.values()) \ + + list(formaters.values()) \ + + list(lifters.values()): + if hasattr(cls, "needs_init"): + initializables.append(cls.__name__) + + if initializables: + parser.add_argument("-i", "--init", + choices = initializables, + help="Initialize the given operator.") asked = parser.parse_args() logging.basicConfig() logger.setLevel("DEBUG") + if asked.init: + for ops in [streamers, consumers, formaters, lifters]: + if asked.init in ops: + logger.debug(f"Initialize operator: {asked.init}") + op = ops[asked.init]() + op.init() + logger.debug(f"Done. You can use the `{asked.init}` operator.") + sys.exit(0) + logger.debug("Available operators:") logger.debug(f"├ streamers: {', '.join(streamers)}") logger.debug(f"├ consumers: {', '.join(consumers)}") From c764b7b0ba897a32b7314879d4ce15b0ffb68f00 Mon Sep 17 00:00:00 2001 From: nojhan Date: Sun, 12 Apr 2026 23:12:51 +0200 Subject: [PATCH 13/13] feat(mastodon): firs working version --- src/forthlift/forthlift.py | 105 ++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/src/forthlift/forthlift.py b/src/forthlift/forthlift.py index fe03c55..2b290bf 100755 --- a/src/forthlift/forthlift.py +++ b/src/forthlift/forthlift.py @@ -87,7 +87,7 @@ class consume: def __call__(self, stream): current = "" for item in stream: - if re.match(self.mark, item[0]): + if re.match(self.mark, item.strip()): yield current if self.skip: current = "" @@ -251,7 +251,7 @@ class lift: class mastodon(Lift): - def __init__(self, dryrun = 'dry'): + def __init__(self, dryrun = 'nodry'): self._scopes = ['read', 'write'] if dryrun == 'dry': @@ -275,23 +275,28 @@ class lift: if self.needs_init(): self.init() + else: + logger.debug( + "│ │ ├ Mastodon lifter logged in: " \ + f"@{self.config['mastodon']['account']}" \ + f"@{self.config['mastodon']['instance']}" \ + ) + logger.debug("│ │ └OK,") self.masto = mastodon.Mastodon( - api_base_url = self.config["instance"], - # client_id = self.config["client_id"], - client_secret = self.config["client_secret"], - access_token = self.config["token"], + api_base_url = self.config["mastodon"]["instance"], + # client_id = self.config["mastodon"]["client_id"], + client_secret = self.config["mastodon"]["client_secret"], + access_token = self.config["mastodon"]["token"], user_agent = f"{self.name}:{self.version}" ) - logger.debug(self.config) - def needs_init(self): - if not self.config["instance"] \ - or not self.config["account"] \ - or not self.config["client_id"] \ - or not self.config["client_secret"] \ - or not self.config["token"]: + if not self.config["mastodon"]["instance"] \ + or not self.config["mastodon"]["account"] \ + or not self.config["mastodon"]["client_id"] \ + or not self.config["mastodon"]["client_secret"] \ + or not self.config["mastodon"]["token"]: return True else: return False @@ -300,8 +305,8 @@ class lift: if not self.needs_init(): print(f"The {__name__} operator has already been initialized") print("Current configuration is:") - print("\tinstance:", self.config["instance"]) - print("\tuser:", self.config["account"]) + print("\tinstance:", self.config["mastodon"]["instance"]) + print("\tuser:", self.config["mastodon"]["account"]) ans = input("Do you want to re-init it? (yes/no): ").strip() if ans[0].lower() == 'n': @@ -315,7 +320,7 @@ class lift: print("For example: https://social.antigene.org") # instance = input("URL: ").strip() instance = "https://social.antigene.org" - self.config["instance"] = instance + self.config["mastodon"]["instance"] = instance self.register_app() self.oauth() @@ -326,42 +331,47 @@ class lift: toml.dump(config, fd) def load_config(self): - logger.debug(f"Load config from: {self.config_path}") + logger.debug(f"│ │ ├ Load config from: {self.config_path}") config = { - "instance": None, - "account": None, - "client_id": None, - "client_secret": None, - "token": None + "mastodon": { + "instance": None, + "account": None, + "client_id": None, + "client_secret": None, + "token": None, + } } local_config = toml.load(self.config_path) config.update(local_config) - logger.debug(config) + + for k,v in config["mastodon"].items(): + logger.debug(f"│ │ │ ├ {k}: {v}") + logger.debug(f"│ │ │ └OK") return config def register_app(self): - logger.debug(f"Register {self.name} on {self.config['instance']}") + logger.debug(f"Register {self.name} on {self.config["mastodon"]['instance']}") client_id, client_secret = mastodon.Mastodon.create_app( self.name, scopes = self._scopes, - api_base_url = self.config["instance"], + api_base_url = self.config['mastodon']["instance"], website = "https://nojhan.net/git/nojhan/forthlift", user_agent = f"{self.name}:{self.version}", ) - self.config["client_id"] = client_id + self.config["mastodon"]["client_id"] = client_id logger.debug(f"ID: {client_id}") - self.config["client_secret"] = client_secret + self.config["mastodon"]["client_secret"] = client_secret logger.debug(f"Secret: {client_secret}") self.save_config(self.config) def oauth(self): - logger.debug(f"OAuth to: {self.config['instance']}") + logger.debug(f"OAuth to: {self.config['mastodon']['instance']}") oauth = mastodon.Mastodon( - client_id = self.config["client_id"], - client_secret = self.config["client_secret"], - api_base_url = self.config["instance"], + client_id = self.config['mastodon']["client_id"], + client_secret = self.config['mastodon']["client_secret"], + api_base_url = self.config['mastodon']["instance"], ) oauth_url = oauth.auth_request_url(scopes = self._scopes) logger.debug(f"Opening web page: {oauth_url}") @@ -375,11 +385,11 @@ class lift: code = oauth_code, scopes = self._scopes, ) - self.config["token"] = token + self.config['mastodon']["token"] = token logger.debug(f"Token: {token}") account = oauth.me().acct - self.config["account"] = account + self.config['mastodon']["account"] = account logger.debug(f"Account: {account}") self.save_config(self.config) @@ -387,32 +397,28 @@ class lift: if self.dry_run: print(item) else: - print(item) + if prev_status: + logger.debug(item) + return self.masto.status_reply(to_status = prev_status, status = item) + else: + logger.debug(item) + return self.masto.status_post(item) def __call__(self, items): n = 0 first_item = next(items, None) if first_item == None: - logger.error("No item to post") + logger.error("│ │ │ ├ No item to post") return else: - logger.debug(f"Post {n+1}") + logger.debug(f"│ │ │ ├ Post #{n+1}") n += 1 - # prev_status = self.masto.status_post( - # status = first_item, - # ) - # logger.debug(prev_status) - self.post(first_item) + prev_status = self.post(first_item) for item in items: - logger.debug(f"Post {n+1}") + logger.debug(f"│ │ │ ├ Post #{n+1}") n += 1 - # prev_status = self.masto.status_reply( - # to_status = prev_status, - # status = item, - # ) - # logger.debug(prev_status) - self.post(item) - logger.debug(f"Posted {n} items") + prev_status = self.post(item) + logger.debug(f"│ │ │ └OK, posted {n} items") class Forthlifter: def __init__(self, @@ -464,6 +470,7 @@ class Forthlifter: logger.debug(f"│ │ ├ {type(lifter).__name__}") # Call lifter(items) + logger.debug(f"│ │ └OK") def __call__(self): logger.debug("├ call")