From 016852e281f86acf306ceb14f834a2eaf5a1c4c9 Mon Sep 17 00:00:00 2001 From: nojhan Date: Wed, 31 Aug 2022 00:24:52 +0200 Subject: [PATCH] adds python implementations - more README - fix some C++ code along the way --- README.md | 76 ++++++++++++++++++++++++++++++++-------- pcat.py | 13 ++++--- run_service1.sh | 3 +- service1.cpp | 2 +- service1.py | 21 +++++++++++ service2.cpp | 34 +++++++----------- service2.py | 93 +++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 199 insertions(+), 43 deletions(-) create mode 100755 service1.py create mode 100755 service2.py diff --git a/README.md b/README.md index f55b199..d9a82e4 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,61 @@ Named pipes services ==================== -Examples of how to design services that use Linux' named pipes FIFO as I/O. +Examples (in C++ and Python) of how to design services that use named pipes FIFO as I/O. + +Instead of implementing heavy web services or complex low-level network code, +just read/write from/to files, and be done with it. -Rationale ---------- +Introduction +------------ + +### Rationale + +The problem of making two programs *communicate* is among the ones +that generated the largest litterature and code in all computer science +(along with cache invalidation, naming things, and web frameworks). + +When facing such a problem, a programer immediatly thinks "I'll use middleware". +If you don't really now what a middleware is, be at ease, nobody really knows. +Nowadays, you may have eared of their latest avatar: *web services*. +As our programmer is going to realize, one now have *two* problems. +The burden of writing, using, and maintaining code using middleware is always huge. +Because they are made to handle a **tremendous** number of complex situations, +most of which involve adversary users, users being bots, or both. + +But most of the time, the actual problem does not really involve these situations. +At least not at the beginning (which means probably never). +If you are building up (firsts versions of) communicating programs +that will run on a (safe) local network, +and for which the exchanged messages are known, +then I have good news: +**you don't have to use web services** (or any kind of middleware). + +**You just need to know how to read/write from/to (special) files**. + + +### Overview The basic idea is that, instead of programming the network interface to your service with low level sockets or any high level library, -you can just implement query/answer mechanisms using named pipes. +you can just implement query/answer mechanisms using **named pipes**. -Named pipes are special FIFO files that are blocking on I/O +Named pipes are special *FIFO* files that are blocking on I/O and implements a very basic form of message passing, without having to bother with polling. -Moreover, they are very easy to use, are they are just files +Moreover, they are very easy to use, as they are just files in which you read/write. Once you made your service on top of named pipes, it is easy to wrap it within an interface made with other languages/tools. -For instance, it is very easy to expose it on the network using common Linux tools like `socat`. +For instance, it is very easy to expose it on the network using common tools like `socat`. Be warned that this is not secure, though, you should only use this for testing -purpose in a secure local network. +purpose in a secured local network. -Principle ---------- +### Principle The theoretical principle can be represented by this UML sequence diagram: ``` @@ -48,24 +77,40 @@ The theoretical principle can be represented by this UML sequence diagram: │ │ │ │ ``` -Note that the service is started first and is waiting for the input. -Note also that there are two pipes, here: one for the input and one for the output. +Notes: +- the service is started first and is waiting for the input, + but as processes are blocking, the starting order does not always matter. +- there are two pipes, here (one for the input and one for the output), + for the sake of simplicity, but you may just as well use only one. Build and run ------------- +Python code does not need to be built. + +To build the C++ code on Linux, just call: ```sh ./build.sh +``` + +You may use the `run_*` scripts to see how to run the examples. +For instance, for the most complex one: +``` ./run_service2.sh ``` + Examples -------- -To create the named pipes under Linux, use the `mkfifo` command, as shown in the `build.sh` +To create the named pipes under Linux or MacOS, use the `mkfifo` command, as shown in the `build.sh` script. +Creating named pipes on windows is more complex, you may want to look at the +[related Stack Overflow question](https://stackoverflow.com/questions/3670039/is-it-possible-to-open-a-named-pipe-with-command-line-in-windows) + + ### Trivial example: a `cat` service The `pcat` executable implements a service that reads from a named pipe and @@ -125,6 +170,8 @@ Use `Ctrl-C` to close the remaining `cat` process. Furthermore ----------- +### Expose such services on network + If you want to expose such a service as a network server, just use socat. For example, to get _data_ query from the network for `service1`: @@ -151,8 +198,7 @@ socat TCP-LISTEN:8478,reuseaddr,fork PIPE:/./data ``` -Troubleshooting -=============== +### Troubleshooting If you witness strange behavior while debugging your own services (like prints that do not occur in the correct terminal), double check that yo don't have some diff --git a/pcat.py b/pcat.py index 9dd0768..1343bf0 100755 --- a/pcat.py +++ b/pcat.py @@ -1,8 +1,11 @@ -#!/usr/bin/env python3 -# +#!/usr/bin/env python + import sys -while True: - with open(sys.argv[1],'r') as fd: - print(fd.read(), flush=True) +if __name__ == "__main__": + + while True: + with open(sys.argv[1]) as fin: + line = fin.readline() + sys.stdout.write(line) diff --git a/run_service1.sh b/run_service1.sh index 5a6d71d..37cc9ec 100755 --- a/run_service1.sh +++ b/run_service1.sh @@ -1,5 +1,6 @@ -./service1 data > out & +# ./service1 data > out & +./service1 data out & PID_SERVICE=$! echo "Hellow World!" > data & diff --git a/service1.cpp b/service1.cpp index 2209a75..c20596d 100644 --- a/service1.cpp +++ b/service1.cpp @@ -26,9 +26,9 @@ int main(int argc, char** argv) ifs.close(); std::string data = strip(datas.str()); + std::clog << "Received: <" << data << ">" << std::endl; std::ofstream ofs(argv[2]); - std::clog << "Received: <" << data << ">" << std::endl; ofs << data << std::endl; ofs.close(); diff --git a/service1.py b/service1.py new file mode 100755 index 0000000..c47ddad --- /dev/null +++ b/service1.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +import sys + +if __name__ == "__main__": + print("Start server") + + while True: + with open(sys.argv[1]) as fin: + datas = fin.readline() + + data = datas.strip() + print("Received: <",data,">", file=sys.stderr) + + with open(sys.argv[2], 'w') as fout: + fout.write(data) + + if data == "exit": + break + + print("Stop server", file=sys.stderr) diff --git a/service2.cpp b/service2.cpp index 1a3dbea..e1bde1b 100644 --- a/service2.cpp +++ b/service2.cpp @@ -8,10 +8,9 @@ #include -enum ERROR { NOT_FIFO=1, NO_CONTEXT }; +enum ERROR { NOT_FIFO=1 }; -class Service -{ +class Service { protected: bool _has_current_context; std::mutex _mutex; @@ -20,14 +19,11 @@ protected: std::string _out; std::string _current_context; - bool has_current_context() - { - std::lock_guard guarded_scope(_mutex); + bool has_current_context() const { return _has_current_context; } - void has_current_context(bool flag) - { + void has_current_context(bool flag) { std::lock_guard guarded_scope(_mutex); _has_current_context = flag; } @@ -45,16 +41,14 @@ public: _out(out) {} - std::string strip(std::string s) - { + std::string strip(std::string s) const { s.erase(std::find_if( s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); } ).base(), s.end()); return s; } - void update_current_context() - { + void update_current_context() { while(true) { std::clog << "Wait for context..." << std::endl; bool has_error = false; @@ -74,8 +68,7 @@ public: } } - void handle_data() - { + void handle_data() const { while(true) { if(this->has_current_context()) { std::string data; @@ -104,13 +97,12 @@ public: out.close(); std::clog << "\tdone" << std::endl; } // if not has_error - } + } // if has context } // while true } }; -bool is_named_pipe_fifo(char* filename) -{ +bool is_named_pipe_fifo(char* filename) { struct stat st; stat(filename, &st); if(not S_ISFIFO(st.st_mode) ) { @@ -119,11 +111,11 @@ bool is_named_pipe_fifo(char* filename) return true; } -int main(int argc, char** argv) -{ - assert(argc = 3); +int main(int argc, char** argv) { - for(size_t i=1; i < 3; ++i) { + assert(argc == 4); + + for(size_t i=1; i < 4; ++i) { if( not is_named_pipe_fifo(argv[i]) ) { std::cerr << "ERROR: " << argv[i] << " is not a named pipe FIFO" << std::endl; exit(ERROR::NOT_FIFO); diff --git a/service2.py b/service2.py new file mode 100755 index 0000000..0fd851f --- /dev/null +++ b/service2.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +from enum import Enum +import threading +import stat +import sys +import os + +class ERROR(Enum): + NOT_FIFO = 1 + +class Service: + def __init__(self, context: str, data: str, out: str) -> None: + self._has_current_context: bool = False + self._mutex = threading.Lock() + self._file_current_context: str = context + self._file_data: str = data + self._out: str = out + self._current_context: str = "" + + def get_has_current_context(self) -> bool: + return self._has_current_context + + def set_has_current_context(self, flag: bool) -> None: + self._mutex.acquire() + self._has_current_context = flag + self._mutex.release() + + def update_current_context(self) -> None: + while True: + print("Wait for context...", file = sys.stderr) + has_error: bool = False + try: + with open(self._file_current_context) as if_current_context: + self._current_context: str = if_current_context.readline().strip() + except: + has_error = True + + if not has_error: + self.set_has_current_context(True) + print("\tReceived context:", self._current_context, file=sys.stderr) + + def handle_data(self) -> None: + while True: + if self.get_has_current_context(): + print("Wait for data...", file=sys.stderr) + has_error: bool = False + try: + with open(self._file_data) as if_data: + data: str = if_data.readline().strip() + except: + has_error = True + + if not has_error: + print("\tReceived data:",data, file=sys.stderr) + + print("Do stuff...", file=sys.stderr) + result: str = self._current_context + ":" + data + print("\tdone", file=sys.stderr) + + print("Output...", file=sys.stderr) + with open(self._out, 'w') as out: + out.write(result) + + print("\tdone", file=sys.stderr) + +def is_named_pipe_fifo(filename: str): + st = os.stat(filename) + return stat.S_ISFIFO(st.st_mode) + +if __name__ == "__main__": + + assert(len(sys.argv) == 4) + + for i in range(1,4): + if not is_named_pipe_fifo(sys.argv[i]): + print("ERROR:", sys.argv[i], "is not a named pipe FIFO", file=sys.stderr) + sys.exit(ERROR.NO_FIFO) + + print("Start server", file=sys.stderr, flush=True) + server = Service(sys.argv[1], sys.argv[2], sys.argv[3]) + + do_current_context = threading.Thread( target = server.update_current_context ) + do_tasks = threading.Thread( target = server.handle_data ) + + do_current_context.start() + do_tasks.start() + + do_current_context.join() + do_tasks.join() + + print("End", file=sys.stderr) +