diff --git a/COPYING b/COPYING index d60c31a..d159169 100644 --- a/COPYING +++ b/COPYING @@ -1,12 +1,12 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - Preamble + Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public @@ -15,7 +15,7 @@ software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to +the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not @@ -55,8 +55,8 @@ patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. - - GNU GENERAL PUBLIC LICENSE + + GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains @@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions: License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - + These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in @@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - + 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is @@ -225,7 +225,7 @@ impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - + 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License @@ -255,7 +255,7 @@ make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. - NO WARRANTY + NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN @@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it @@ -303,17 +303,16 @@ the "copyright" line and a pointer to where the full notice is found. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: - Gnomovision version 69, Copyright (C) year name of author + Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. @@ -336,5 +335,5 @@ necessary. Here is a sample; alter the names: This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General +library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. diff --git a/capture b/capture deleted file mode 100755 index e0d32d4..0000000 --- a/capture +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -# example of capturing target data for smem - -# capture a memory data snapshot with realtime priority -mkdir -p $1 -chrt --fifo 99 \ - cp -a --parents /proc/[0-9]*/{smaps,cmdline,stat} /proc/meminfo /proc/version $1 - -# build a compressed tarball of snapshot -cd $1/proc -tar czf ../../$1.tgz * - diff --git a/smem b/smem index 66e7a91..7020a27 100755 --- a/smem +++ b/smem @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # smem - a tool for meaningful memory reporting # @@ -8,17 +8,20 @@ # the GNU General Public License version 2 or later, incorporated # herein by reference. -import re, os, sys, pwd, grp, optparse, errno, tarfile +import re, os, sys, pwd, optparse, errno, tarfile + +warned = False class procdata(object): def __init__(self, source): self._ucache = {} self._gcache = {} self.source = source and source or "" + self._memdata = None def _list(self): return os.listdir(self.source + "/proc") def _read(self, f): - return file(self.source + '/proc/' + f).read() + return open(self.source + '/proc/' + f).read() def _readlines(self, f): return self._read(f).splitlines(True) def _stat(self, f): @@ -31,26 +34,50 @@ class procdata(object): def mapdata(self, pid): return self._readlines('%s/smaps' % pid) def memdata(self): - return self._readlines('meminfo') + if self._memdata is None: + self._memdata = self._readlines('meminfo') + return self._memdata def version(self): return self._readlines('version')[0] def pidname(self, pid): - l = self._read('%d/stat' % pid) - return l[l.find('(') + 1: l.find(')')] + try: + l = self._read('%d/stat' % pid) + return l[l.find('(') + 1: l.find(')')] + except: + return '?' def pidcmd(self, pid): - c = self._read('%s/cmdline' % pid)[:-1] - return c.replace('\0', ' ') + try: + c = self._read('%s/cmdline' % pid)[:-1] + return c.replace('\0', ' ') + except: + return '?' def piduser(self, pid): - return self._stat('%d/cmdline' % pid).st_uid + try: + return self._stat('%d' % pid).st_uid + except: + return -1 def pidgroup(self, pid): - return self._stat('%d/cmdline' % pid).st_gid + try: + return self._stat('%d' % pid).st_gid + except: + return -1 def username(self, uid): + if uid == -1: + return '?' if uid not in self._ucache: - self._ucache[uid] = pwd.getpwuid(uid)[0] + try: + self._ucache[uid] = pwd.getpwuid(uid)[0] + except KeyError: + self._ucache[uid] = str(uid) return self._ucache[uid] def groupname(self, gid): + if gid == -1: + return '?' if gid not in self._gcache: - self._gcache[gid] = pwd.getgrgid(gid)[0] + try: + self._gcache[gid] = pwd.getgrgid(gid)[0] + except KeyError: + self._gcache[gid] = str(gid) return self._gcache[gid] class tardata(procdata): @@ -67,12 +94,12 @@ class tardata(procdata): def _readlines(self, f): return self.tar.extractfile(f).readlines() def piduser(self, p): - t = self.tar.getmember("%s/cmdline" % p) + t = self.tar.getmember("%d" % p) if t.uname: self._ucache[t.uid] = t.uname return t.uid def pidgroup(self, p): - t = self.tar.getmember("%s/cmdline" % p) + t = self.tar.getmember("%d" % p) if t.gname: self._gcache[t.gid] = t.gname return t.gid @@ -99,17 +126,32 @@ def kernelsize(): d = os.popen("size %s" % options.kernel).readlines()[1] _kernelsize = int(d.split()[3]) / 1024 except: - pass + try: + # try some heuristic to find gzipped part in kernel image + packedkernel = open(options.kernel).read() + pos = packedkernel.find('\x1F\x8B') + if pos >= 0 and pos < 25000: + sys.stderr.write("Maybe uncompressed kernel can be extracted by the command:\n" + " dd if=%s bs=1 skip=%d | gzip -d >%s.unpacked\n\n" % (options.kernel, pos, options.kernel)) + except: + pass + sys.stderr.write("Parameter '%s' should be an original uncompressed compiled kernel file.\n\n" % options.kernel) return _kernelsize def pidmaps(pid): + global warned maps = {} start = None + seen = False + empty = True for l in src.mapdata(pid): - f = l.split() - if f[-1] == 'kB': + empty = False + f = l.split() + if f[-1] == 'kB': + if f[0].startswith('Pss'): + seen = True maps[start][f[0][:-1].lower()] = int(f[1]) - else: + elif '-' in f[0] and ':' not in f[0]: # looks like a mapping range start, end = f[0].split('-') start = int(start, 16) name = "" @@ -119,10 +161,16 @@ def pidmaps(pid): offset=int(f[2], 16), device=f[3], inode=f[4], name=name) + if not empty and not seen and not warned: + sys.stderr.write('warning: kernel does not appear to support PSS measurement\n') + warned = True + if not options.sort: + options.sort = 'rss' + if options.mapfilter: f = {} for m in maps: - if not filter(options.mapfilter, m, lambda x: maps[x]['name']): + if not filters(options.mapfilter, m, lambda x: maps[x]['name']): f[m] = maps[m] return f @@ -151,7 +199,7 @@ def units(x): s = '' if x == 0: return '0' - for s in ('', 'K', 'M', 'G'): + for s in ('', 'K', 'M', 'G', 'T'): if x < 1024: break x /= 1024.0 @@ -159,22 +207,27 @@ def units(x): def fromunits(x): s = dict(k=2**10, K=2**10, kB=2**10, KB=2**10, - M=2**20, MB=2**20, G=2**30, GB=2**30) + M=2**20, MB=2**20, G=2**30, GB=2**30, + T=2**40, TB=2**40) for k,v in s.items(): if x.endswith(k): - return int(float(x[:len(k)])*v) + return int(float(x[:-len(k)])*v) + sys.stderr.write("Memory size should be written with units, for example 1024M\n") + sys.exit(-1) def pidusername(pid): return src.username(src.piduser(pid)) -def showamount(a): +def showamount(a, total): if options.abbreviate: return units(a * 1024) elif options.percent: - return "%.2f%%" % (100.0 * a / totalmem()) + if total == 0: + return 'N/A' + return "%.2f%%" % (100.0 * a / total) return a -def filter(opt, arg, *sources): +def filters(opt, arg, *sources): if not opt: return False @@ -187,7 +240,7 @@ def pidtotals(pid): maps = pidmaps(pid) t = dict(size=0, rss=0, pss=0, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=0, referenced=0, swap=0) - for m in maps.iterkeys(): + for m in maps: for k in t: t[k] += maps[m].get(k, 0) @@ -199,8 +252,8 @@ def pidtotals(pid): def processtotals(pids): totals = {} for pid in pids: - if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or - filter(options.userfilter, pid, pidusername)): + if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or + filters(options.userfilter, pid, pidusername)): continue try: p = pidtotals(pid) @@ -220,7 +273,7 @@ def showpids(): return pidusername(p) fields = dict( - pid=('PID', lambda n: n, '% 5s', lambda x: len(p), + pid=('PID', lambda n: n, '% 5s', lambda x: len(pt), 'process ID'), user=('User', showuser, '%-8s', lambda x: len(dict.fromkeys(x)), 'owner of process'), @@ -248,13 +301,13 @@ def showpids(): def maptotals(pids): totals = {} for pid in pids: - if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or - filter(options.userfilter, pid, pidusername)): + if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or + filters(options.userfilter, pid, pidusername)): continue try: maps = pidmaps(pid) seen = {} - for m in maps.iterkeys(): + for m in maps: name = maps[m]['name'] if name not in totals: t = dict(size=0, rss=0, pss=0, shared_clean=0, @@ -270,8 +323,8 @@ def maptotals(pids): t['pids'] += 1 seen[name] = 1 totals[name] = t - except: - raise + except EnvironmentError: + continue return totals def showmaps(): @@ -313,15 +366,15 @@ def showmaps(): def usertotals(pids): totals = {} for pid in pids: - if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or - filter(options.userfilter, pid, pidusername)): + if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or + filters(options.userfilter, pid, pidusername)): continue try: maps = pidmaps(pid) if len(maps) == 0: continue - except: - raise + except EnvironmentError: + continue user = src.piduser(pid) if user not in totals: t = dict(size=0, rss=0, pss=0, shared_clean=0, @@ -330,7 +383,7 @@ def usertotals(pids): else: t = totals[user] - for m in maps.iterkeys(): + for m in maps: for k in t: t[k] += maps[m].get(k, 0) @@ -410,11 +463,44 @@ def showsystem(): showtable(range(len(l)), fields, columns.split(), options.sort or 'order') def showfields(fields, f): - if f != list: - print "unknown field", f - print "known fields:" - for l in sorted(fields.keys()): - print "%-8s %s" % (l, fields[l][-1]) + if type(f) in (list, set): + print("unknown fields: " + " ".join(f)) + else: + print("unknown field %s" % f) + print("known fields:") + for l in sorted(fields): + print("%-8s %s" % (l, fields[l][-1])) + +def autosize(columns, fields, rows): + colsizes = {} + for c in columns: + sizes = [1] + + if not options.no_header: + sizes.append(len(fields[c][0])) + + if (options.abbreviate or options.percent) and 'a' in fields[c][2]: + sizes.append(7) + else: + for r in rows: + sizes.append(len(str(fields[c][1](r)))) + + colsizes[c] = max(sizes) + + overflowcols = set(["command", "map"]) & set(columns) + if len(overflowcols) > 0: + overflowcol = overflowcols.pop() + totnoflow = sum(colsizes.values()) - colsizes[overflowcol] + try: + ttyrows, ttycolumns = os.popen('stty size', 'r').read().split() + ttyrows, ttycolumns = int(ttyrows), int(ttycolumns) + except: + ttyrows, ttycolumns = (24, 80) + maxflowcol = ttycolumns - totnoflow - len(columns) + maxflowcol = max(maxflowcol, 10) + colsizes[overflowcol] = min(colsizes[overflowcol], maxflowcol) + + return colsizes def showtable(rows, fields, columns, sort): header = "" @@ -430,17 +516,31 @@ def showtable(rows, fields, columns, sort): if options.bar: columns.append(options.bar) - for n in columns: - if n not in fields: - showfields(fields, n) - sys.exit(-1) + mt = totalmem() + st = memory()['swaptotal'] + missing = set(columns) - set(fields) + if len(missing) > 0: + showfields(fields, missing) + sys.exit(-1) + + if options.autosize: + colsizes = autosize(columns, fields, rows) + else: + colsizes = {} + + for n in columns: f = fields[n][2] if 'a' in f: - formatter.append(showamount) + if n == 'swap': + formatter.append(lambda x: showamount(x, st)) + else: + formatter.append(lambda x: showamount(x, mt)) f = f.replace('a', 's') else: formatter.append(lambda x: x) + if n in colsizes: + f = re.sub(r"[0-9]+", str(colsizes[n]), f) format += f + " " header += f % fields[n][0] + " " @@ -459,10 +559,10 @@ def showtable(rows, fields, columns, sort): return if not options.no_header: - print header + print(header) for k,r in l: - print format % tuple([f(v) for f,v in zip(formatter, r)]) + print(format % tuple([f(v) for f,v in zip(formatter, r)])) if options.totals: # totals @@ -474,8 +574,8 @@ def showtable(rows, fields, columns, sort): else: t.append("") - print "-" * len(header) - print format % tuple([f(v) for f,v in zip(formatter, t)]) + print("-" * len(header)) + print(format % tuple([f(v) for f,v in zip(formatter, t)])) def showpie(l, sort): try: @@ -494,15 +594,12 @@ def showpie(l, sort): s = sum(values) unused = tm - s t = 0 - c = 0 - while values and (t + values[-1 - c] < (tm * .02) or - values[-1 - c] < (tm * .005)): - c += 1 + while values and (t + values[-1] < (tm * .02) or + values[-1] < (tm * .005)): t += values.pop() labels.pop() - if c > 1: - values = values[:-c] - labels = labels[:-c] + + if t: values.append(t) labels.append('other') @@ -559,12 +656,6 @@ def showbar(l, columns, sort): pylab.legend([p[0] for p in pl], key) pylab.show() -def kernel_version_check(): - kernel_release = src.version().split()[2].split('-')[0] - if kernel_release < "2.6.27": - name = os.path.basename(sys.argv[0]) - sys.stderr.write(name + " requires a kernel >= 2.6.27\n") - sys.exit(-1) parser = optparse.OptionParser("%prog [options]") parser.add_option("-H", "--no-header", action="store_true", @@ -573,6 +664,8 @@ parser.add_option("-c", "--columns", type="str", help="columns to show") parser.add_option("-t", "--totals", action="store_true", help="show totals") +parser.add_option("-a", "--autosize", action="store_true", + help="size columns to fit terminal size") parser.add_option("-R", "--realmem", type="str", help="amount of physical RAM") @@ -623,8 +716,6 @@ try: except: src = procdata(options.source) -kernel_version_check() - try: if options.mappings: showmaps() @@ -634,7 +725,8 @@ try: showsystem() else: showpids() -except IOError, e: +except IOError: + _, e, _ = sys.exc_info() if e.errno == errno.EPIPE: pass except KeyboardInterrupt: diff --git a/smem.8 b/smem.8 index b66c789..b71bd32 100644 --- a/smem.8 +++ b/smem.8 @@ -1,4 +1,4 @@ -.TH SMEM 8 "07/09/2009" "" "" +.TH SMEM 8 "03/15/2010" "" "" .SH NAME smem \- Report memory usage with shared memory divided proportionally. @@ -16,7 +16,7 @@ is reported as the PSS (Proportional Set Size). The USS and PSS only include physical memory usage. They do not include memory that has been swapped out to disk. -Memory can be reported by process, by user, by mapping, or system\-wide. +Memory can be reported by process, by user, by mapping, or systemwide. Both text mode and graphical output are available. .SH OPTIONS @@ -34,16 +34,21 @@ you used a tarred set of /proc data saved earlier, possibly on a different machine. The \-\-kernel and \-\-realmem options let you specify a couple things that smem cannot discover on its own. + .TP .BI "\-K " KERNEL ", \-\-kernel=" KERNEL -Path to kernel image. This lets smem include the size of the kernel's -code and statically allocated data in the systemwide (-w) output. +Path to an uncompressed kernel image. This lets smem include the size +of the kernel's code and statically allocated data in the systemwide +(\-w) output. (To obtain an uncompressed image of a kernel on disk, you +may need to build the kernel yourself, then locate file vmlinux in the +source tree.) .TP .BI "\-R " REALMEM ", \-\-realmem=" REALMEM Amount of physical RAM. This lets smem detect the amount of memory used -by firmware/hardware in the systemwide (-w) output. If provided, it +by firmware/hardware in the systemwide (\-w) output. If provided, it will also be used as the total memory size to base percentages on. +Example: --realmem=1024M .TP .BI "\-S " SOURCE ", \-\-source=" SOURCE @@ -54,7 +59,7 @@ using smemcap, and parse the data later on a different machine. If the running system. .SS REPORT BY -If none of the following options are include, smem reports memory usage +If none of the following options are included, smem reports memory usage by process. .TP @@ -90,6 +95,10 @@ User filter regular expression. .SS OUTPUT FORMATTING +.TP +.B \-a, \-\-autosize +Size columns to fit terminal size. + .TP .BI "\-c " COLUMNS ", \-\-columns=" COLUMNS Columns to show. @@ -135,8 +144,39 @@ Show pie graph. .PP -.SH NOTES -\fBsmem\fP requires a 2.6.27 or newer kernel. +.SH REQUIREMENTS +\fBsmem\fP requires: + +.IP \(bu 3 +Linux kernel providing 'Pss' metric in /proc//smaps (generally +2.6.27 or newer). +.IP \(bu +Python 2.x (at least 2.4 or so). +.IP \(bu +The matplotlib library +(only if you want to generate graphical charts). + +.SH EMBEDDED USAGE +To capture memory statistics on resource\-constrained systems, the +the \fBsmem\fP source includes a utility named \fBsmemcap\fP. +\fBsmemcap\fP captures all /proc entries required by \fBsmem\fP +and outputs them as an uncompressed .tar file to STDOUT. +\fBsmem\fP can analyze the output using the \fB\-\-source\fP option. +\fBsmemcap\fP is small and does not require Python. +.PP +To use \fBsmemcap\fP: +.IP 1. 3 +Obtain the smem source at http://selenic.com/repo/smem +.IP 2. +Compile \fIsmemcap.c\fP for your target system. +.IP 3. +Run \fBsmemcap\fP on the target system and save the output: +.br +smemcap > memorycapture.tar +.IP 4. +Copy the output to another machine and run smem on it: +.br +smem -S memorycapture.tar .SH FILES .I /proc/$pid/cmdline @@ -149,8 +189,25 @@ Show pie graph. .PP .I /proc/version +.SH RESOURCES +Main Web Site: http://www.selenic.com/smem + +Source code repository: http://selenic.com/repo/smem + +Mailing list: http://selenic.com/mailman/listinfo/smem + .SH "SEE ALSO" -.BR free (1), pmap (1) +.BR free (1), +.BR pmap (1), +.BR proc (5), +.BR ps (1), +.BR top (1), +.BR vmstat (8) + +.SH COPYING +Copyright (C) 2008-2009 Matt Mackall. Free use of this software +is granted under the terms of the GNU General Public License +version 2 or later. .SH AUTHOR \fBsmem\fP was written by Matt Mackall. diff --git a/smemcap.c b/smemcap.c index d8af86f..641c63b 100644 --- a/smemcap.c +++ b/smemcap.c @@ -32,17 +32,17 @@ int writeheader(int destfd, const char *path, int mode, int uid, int gid, memset(header, 0, 512); sprintf(header, "%s", path); - sprintf(header + 100, "%07o\0", mode & 0777); - sprintf(header + 108, "%07o\0", uid); - sprintf(header + 116, "%07o\0", gid); - sprintf(header + 124, "%011o\0", size); - sprintf(header + 136, "%07o\0", mtime); + sprintf(header + 100, "%07o", mode & 0777); + sprintf(header + 108, "%07o", uid); + sprintf(header + 116, "%07o", gid); + sprintf(header + 124, "%011o", size); + sprintf(header + 136, "%07o", mtime); sprintf(header + 148, " %1d", type); /* fix checksum */ for (i = sum = 0; i < 512; i++) - sum += header[i]; - sprintf(header + 148, "%06o\0 %1d", sum, type); + sum += header[i]; + sprintf(header + 148, "%06o", sum); return write(destfd, header, 512); } @@ -91,18 +91,20 @@ int archivejoin(const char *sub, const char *name, int destfd) int main(int argc, char *argv[]) { - int fd; DIR *d; struct dirent *de; + struct stat s; chdir("/proc"); archivefile("meminfo", 1); archivefile("version", 1); d = opendir("."); - while (de = readdir(d)) + while ((de = readdir(d))) if (de->d_name[0] >= '0' && de->d_name[0] <= '9') { - writeheader(1, de->d_name, 0555, 0, 0, 0, 0, 5); + stat (de->d_name, &s); + writeheader(1, de->d_name, 0555, s.st_uid, + s.st_gid, 0, s.st_mtime, 5); archivejoin(de->d_name, "smaps", 1); archivejoin(de->d_name, "cmdline", 1); archivejoin(de->d_name, "stat", 1);