diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/capture b/capture deleted file mode 100755 index e23995c..0000000 --- a/capture +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -# example of capturing target data for smem - -# capture a memory data snapshot with realtime priority -mkdir -p $1 -chrt --fifo 99 \ - cp -a --parents /proc/[0-9]*/{smaps,cmdline,stat} /proc/meminfo $1 - -# build a compressed tarball of snapshot -cd $1/proc -tar czf ../../$1.tgz * - diff --git a/smem b/smem index c4cffd8..7020a27 100755 --- a/smem +++ b/smem @@ -1,16 +1,27 @@ -#!/usr/bin/python -import re, os, sys, pwd, grp, optparse, errno, tarfile +#!/usr/bin/env python +# +# smem - a tool for meaningful memory reporting +# +# Copyright 2008-2009 Matt Mackall +# +# This software may be used and distributed according to the terms of +# the GNU General Public License version 2 or later, incorporated +# herein by reference. -_ucache = {} -_gcache = {} +import re, os, sys, pwd, optparse, errno, tarfile + +warned = False class procdata(object): def __init__(self, source): + self._ucache = {} + self._gcache = {} self.source = source and source or "" + self._memdata = None def _list(self): return os.listdir(self.source + "/proc") def _read(self, f): - return file(self.source + '/proc/' + f).read() + return open(self.source + '/proc/' + f).read() def _readlines(self, f): return self._read(f).splitlines(True) def _stat(self, f): @@ -23,69 +34,124 @@ class procdata(object): def mapdata(self, pid): return self._readlines('%s/smaps' % pid) def memdata(self): - return self._readlines('meminfo') + if self._memdata is None: + self._memdata = self._readlines('meminfo') + return self._memdata + def version(self): + return self._readlines('version')[0] def pidname(self, pid): - l = self._read('%d/stat' % pid) - return l[l.find('(') + 1: l.find(')')] + try: + l = self._read('%d/stat' % pid) + return l[l.find('(') + 1: l.find(')')] + except: + return '?' def pidcmd(self, pid): - c = self._read('%s/cmdline' % pid)[:-1] - return c.replace('\0', ' ') + try: + c = self._read('%s/cmdline' % pid)[:-1] + return c.replace('\0', ' ') + except: + return '?' def piduser(self, pid): - return self._stat('%d/cmdline' % pid).st_uid + try: + return self._stat('%d' % pid).st_uid + except: + return -1 def pidgroup(self, pid): - return self._stat('%d/cmdline' % pid).st_gid + try: + return self._stat('%d' % pid).st_gid + except: + return -1 def username(self, uid): - if uid not in _ucache: - _ucache[uid] = pwd.getpwuid(uid)[0] - return _ucache[uid] + if uid == -1: + return '?' + if uid not in self._ucache: + try: + self._ucache[uid] = pwd.getpwuid(uid)[0] + except KeyError: + self._ucache[uid] = str(uid) + return self._ucache[uid] def groupname(self, gid): - if gid not in _gcache: - _gcache[gid] = pwd.getgrgid(gid)[0] - return _gcache[gid] + if gid == -1: + return '?' + if gid not in self._gcache: + try: + self._gcache[gid] = pwd.getgrgid(gid)[0] + except KeyError: + self._gcache[gid] = str(gid) + return self._gcache[gid] class tardata(procdata): def __init__(self, source): - self.source = source + procdata.__init__(self, source) self.tar = tarfile.open(source) def _list(self): for ti in self.tar: - if ti.name.endswith('/'): - yield ti.name[:-1] + if ti.name.endswith('/smaps'): + d,f = ti.name.split('/') + yield d def _read(self, f): return self.tar.extractfile(f).read() def _readlines(self, f): return self.tar.extractfile(f).readlines() def piduser(self, p): - return self.tar.getmember("%s/cmdline" % p).uid + t = self.tar.getmember("%d" % p) + if t.uname: + self._ucache[t.uid] = t.uname + return t.uid def pidgroup(self, p): - return self.tar.getmember("%s/cmdline" % p).gid + t = self.tar.getmember("%d" % p) + if t.gname: + self._gcache[t.gid] = t.gname + return t.gid + def username(self, u): + return self._ucache.get(u, str(u)) + def groupname(self, g): + return self._gcache.get(g, str(g)) _totalmem = 0 def totalmem(): global _totalmem if not _totalmem: if options.realmem: - _totalmem = fromunits(options.realmem) + _totalmem = fromunits(options.realmem) / 1024 else: _totalmem = memory()['memtotal'] - return _totalmem / 1024 + return _totalmem _kernelsize = 0 def kernelsize(): global _kernelsize if not _kernelsize and options.kernel: - d = os.popen("size %s" % options.kernel).readlines()[1] - _kernelsize = int(d.split()[3]) / 1024 + try: + d = os.popen("size %s" % options.kernel).readlines()[1] + _kernelsize = int(d.split()[3]) / 1024 + except: + try: + # try some heuristic to find gzipped part in kernel image + packedkernel = open(options.kernel).read() + pos = packedkernel.find('\x1F\x8B') + if pos >= 0 and pos < 25000: + sys.stderr.write("Maybe uncompressed kernel can be extracted by the command:\n" + " dd if=%s bs=1 skip=%d | gzip -d >%s.unpacked\n\n" % (options.kernel, pos, options.kernel)) + except: + pass + sys.stderr.write("Parameter '%s' should be an original uncompressed compiled kernel file.\n\n" % options.kernel) return _kernelsize def pidmaps(pid): + global warned maps = {} start = None + seen = False + empty = True for l in src.mapdata(pid): - f = l.split() - if f[-1] == 'kB': + empty = False + f = l.split() + if f[-1] == 'kB': + if f[0].startswith('Pss'): + seen = True maps[start][f[0][:-1].lower()] = int(f[1]) - else: + elif '-' in f[0] and ':' not in f[0]: # looks like a mapping range start, end = f[0].split('-') start = int(start, 16) name = "" @@ -95,10 +161,16 @@ def pidmaps(pid): offset=int(f[2], 16), device=f[3], inode=f[4], name=name) + if not empty and not seen and not warned: + sys.stderr.write('warning: kernel does not appear to support PSS measurement\n') + warned = True + if not options.sort: + options.sort = 'rss' + if options.mapfilter: f = {} for m in maps: - if not filter(options.mapfilter, m, lambda x: maps[x]['name']): + if not filters(options.mapfilter, m, lambda x: maps[x]['name']): f[m] = maps[m] return f @@ -127,7 +199,7 @@ def units(x): s = '' if x == 0: return '0' - for s in ('', 'K', 'M', 'G'): + for s in ('', 'K', 'M', 'G', 'T'): if x < 1024: break x /= 1024.0 @@ -135,34 +207,27 @@ def units(x): def fromunits(x): s = dict(k=2**10, K=2**10, kB=2**10, KB=2**10, - M=2**20, MB=2**20, G=2**30, GB=2**30) + M=2**20, MB=2**20, G=2**30, GB=2**30, + T=2**40, TB=2**40) for k,v in s.items(): if x.endswith(k): - return int(float(x[:len(k)])*v) - -_ucache = {} -def username(uid): - if uid not in _ucache: - _ucache[uid] = pwd.getpwuid(uid)[0] - return _ucache[uid] + return int(float(x[:-len(k)])*v) + sys.stderr.write("Memory size should be written with units, for example 1024M\n") + sys.exit(-1) def pidusername(pid): - return username(src.piduser(pid)) + return src.username(src.piduser(pid)) -_gcache = {} -def groupname(gid): - if gid not in _gcache: - _gcache[gid] = grp.getgrgid(gid)[0] - return _gcache[gid] - -def showamount(a): +def showamount(a, total): if options.abbreviate: return units(a * 1024) elif options.percent: - return "%.2f%%" % (100.0 * a / totalmem()) + if total == 0: + return 'N/A' + return "%.2f%%" % (100.0 * a / total) return a -def filter(opt, arg, *sources): +def filters(opt, arg, *sources): if not opt: return False @@ -175,7 +240,7 @@ def pidtotals(pid): maps = pidmaps(pid) t = dict(size=0, rss=0, pss=0, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=0, referenced=0, swap=0) - for m in maps.iterkeys(): + for m in maps: for k in t: t[k] += maps[m].get(k, 0) @@ -187,8 +252,8 @@ def pidtotals(pid): def processtotals(pids): totals = {} for pid in pids: - if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or - filter(options.userfilter, pid, pidusername)): + if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or + filters(options.userfilter, pid, pidusername)): continue try: p = pidtotals(pid) @@ -204,11 +269,11 @@ def showpids(): def showuser(p): if options.numeric: - return piduser(p) + return src.piduser(p) return pidusername(p) fields = dict( - pid=('PID', lambda n: n, '% 5s', lambda x: len(p), + pid=('PID', lambda n: n, '% 5s', lambda x: len(pt), 'process ID'), user=('User', showuser, '%-8s', lambda x: len(dict.fromkeys(x)), 'owner of process'), @@ -236,13 +301,13 @@ def showpids(): def maptotals(pids): totals = {} for pid in pids: - if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or - filter(options.userfilter, pid, pidusername)): + if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or + filters(options.userfilter, pid, pidusername)): continue try: maps = pidmaps(pid) seen = {} - for m in maps.iterkeys(): + for m in maps: name = maps[m]['name'] if name not in totals: t = dict(size=0, rss=0, pss=0, shared_clean=0, @@ -258,8 +323,8 @@ def maptotals(pids): t['pids'] += 1 seen[name] = 1 totals[name] = t - except: - raise + except EnvironmentError: + continue return totals def showmaps(): @@ -301,15 +366,15 @@ def showmaps(): def usertotals(pids): totals = {} for pid in pids: - if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or - filter(options.userfilter, pid, pidusername)): + if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or + filters(options.userfilter, pid, pidusername)): continue try: maps = pidmaps(pid) if len(maps) == 0: continue - except: - raise + except EnvironmentError: + continue user = src.piduser(pid) if user not in totals: t = dict(size=0, rss=0, pss=0, shared_clean=0, @@ -318,7 +383,7 @@ def usertotals(pids): else: t = totals[user] - for m in maps.iterkeys(): + for m in maps: for k in t: t[k] += maps[m].get(k, 0) @@ -333,7 +398,7 @@ def showusers(): def showuser(u): if options.numeric: return u - return username(u) + return src.username(u) fields = dict( user=('User', showuser, '%-8s', None, @@ -358,26 +423,34 @@ def showusers(): def showsystem(): t = totalmem() - k = kernelsize() + ki = kernelsize() m = memory() + mt = m['memtotal'] - fh = t - mt - k f = m['memfree'] + + # total amount used by hardware + fh = max(t - mt - ki, 0) + + # total amount mapped into userspace (ie mapped an unmapped pages) u = m['anonpages'] + m['mapped'] + + # total amount allocated by kernel not for userspace kd = mt - f - u - kdd = (m['buffers'] + m['sreclaimable'] + - (m['cached'] - m['anonpages'])) + + # total amount in kernel caches + kdc = m['buffers'] + m['sreclaimable'] + (m['cached'] - m['mapped']) l = [("firmware/hardware", fh, 0), - ("kernel image", k, 0), - ("kernel dynamic memory", kd, kdd), + ("kernel image", ki, 0), + ("kernel dynamic memory", kd, kdc), ("userspace memory", u, m['mapped']), ("free memory", f, f)] fields = dict( - order=('Order', lambda n: n, '% 1s', lambda x: len(p), + order=('Order', lambda n: n, '% 1s', lambda x: '', 'hierarchical order'), - area=('Area', lambda n: l[n][0], '%-24s', lambda x: len(l), + area=('Area', lambda n: l[n][0], '%-24s', lambda x: '', 'memory area'), used=('Used', lambda n: l[n][1], '%10a', sum, 'area in use'), @@ -390,11 +463,44 @@ def showsystem(): showtable(range(len(l)), fields, columns.split(), options.sort or 'order') def showfields(fields, f): - if f != list: - print "unknown field", f - print "known fields:" - for l in sorted(fields.keys()): - print "%-8s %s" % (l, fields[l][-1]) + if type(f) in (list, set): + print("unknown fields: " + " ".join(f)) + else: + print("unknown field %s" % f) + print("known fields:") + for l in sorted(fields): + print("%-8s %s" % (l, fields[l][-1])) + +def autosize(columns, fields, rows): + colsizes = {} + for c in columns: + sizes = [1] + + if not options.no_header: + sizes.append(len(fields[c][0])) + + if (options.abbreviate or options.percent) and 'a' in fields[c][2]: + sizes.append(7) + else: + for r in rows: + sizes.append(len(str(fields[c][1](r)))) + + colsizes[c] = max(sizes) + + overflowcols = set(["command", "map"]) & set(columns) + if len(overflowcols) > 0: + overflowcol = overflowcols.pop() + totnoflow = sum(colsizes.values()) - colsizes[overflowcol] + try: + ttyrows, ttycolumns = os.popen('stty size', 'r').read().split() + ttyrows, ttycolumns = int(ttyrows), int(ttycolumns) + except: + ttyrows, ttycolumns = (24, 80) + maxflowcol = ttycolumns - totnoflow - len(columns) + maxflowcol = max(maxflowcol, 10) + colsizes[overflowcol] = min(colsizes[overflowcol], maxflowcol) + + return colsizes def showtable(rows, fields, columns, sort): header = "" @@ -410,17 +516,31 @@ def showtable(rows, fields, columns, sort): if options.bar: columns.append(options.bar) - for n in columns: - if n not in fields: - showfields(fields, n) - sys.exit(-1) + mt = totalmem() + st = memory()['swaptotal'] + missing = set(columns) - set(fields) + if len(missing) > 0: + showfields(fields, missing) + sys.exit(-1) + + if options.autosize: + colsizes = autosize(columns, fields, rows) + else: + colsizes = {} + + for n in columns: f = fields[n][2] if 'a' in f: - formatter.append(showamount) + if n == 'swap': + formatter.append(lambda x: showamount(x, st)) + else: + formatter.append(lambda x: showamount(x, mt)) f = f.replace('a', 's') else: formatter.append(lambda x: x) + if n in colsizes: + f = re.sub(r"[0-9]+", str(colsizes[n]), f) format += f + " " header += f % fields[n][0] + " " @@ -439,10 +559,10 @@ def showtable(rows, fields, columns, sort): return if not options.no_header: - print header + print(header) for k,r in l: - print format % tuple([f(v) for f,v in zip(formatter, r)]) + print(format % tuple([f(v) for f,v in zip(formatter, r)])) if options.totals: # totals @@ -454,8 +574,8 @@ def showtable(rows, fields, columns, sort): else: t.append("") - print "-" * len(header) - print format % tuple([f(v) for f,v in zip(formatter, t)]) + print("-" * len(header)) + print(format % tuple([f(v) for f,v in zip(formatter, t)])) def showpie(l, sort): try: @@ -474,15 +594,12 @@ def showpie(l, sort): s = sum(values) unused = tm - s t = 0 - c = 0 - while values and (t + values[-1 - c] < (tm * .02) or - values[-1 - c] < (tm * .005)): - c += 1 + while values and (t + values[-1] < (tm * .02) or + values[-1] < (tm * .005)): t += values.pop() labels.pop() - if c > 1: - values = values[:-c] - labels = labels[:-c] + + if t: values.append(t) labels.append('other') @@ -539,6 +656,7 @@ def showbar(l, columns, sort): pylab.legend([p[0] for p in pl], key) pylab.show() + parser = optparse.OptionParser("%prog [options]") parser.add_option("-H", "--no-header", action="store_true", help="disable header line") @@ -546,6 +664,8 @@ parser.add_option("-c", "--columns", type="str", help="columns to show") parser.add_option("-t", "--totals", action="store_true", help="show totals") +parser.add_option("-a", "--autosize", action="store_true", + help="size columns to fit terminal size") parser.add_option("-R", "--realmem", type="str", help="amount of physical RAM") @@ -605,7 +725,8 @@ try: showsystem() else: showpids() -except IOError, e: +except IOError: + _, e, _ = sys.exc_info() if e.errno == errno.EPIPE: pass except KeyboardInterrupt: diff --git a/smem.8 b/smem.8 new file mode 100644 index 0000000..b71bd32 --- /dev/null +++ b/smem.8 @@ -0,0 +1,214 @@ +.TH SMEM 8 "03/15/2010" "" "" + +.SH NAME +smem \- Report memory usage with shared memory divided proportionally. + +.SH SYNOPSIS +.B smem +.RI [ options ] + +.SH DESCRIPTION +\fBsmem\fP reports physical memory usage, taking shared memory pages +into account. Unshared memory is reported as the USS (Unique Set Size). +Shared memory is divided evenly among the processes sharing that memory. +The unshared memory (USS) plus a process's proportion of shared memory +is reported as the PSS (Proportional Set Size). The USS and PSS only +include physical memory usage. They do not include memory that has been +swapped out to disk. + +Memory can be reported by process, by user, by mapping, or systemwide. +Both text mode and graphical output are available. + +.SH OPTIONS + +.SS GENERAL OPTIONS + +.TP +.B \-h, \-\-help +Show help. + +.SS SOURCE DATA +By default, smem will pull most of the data it needs from the /proc +filesystem of the system it is running on. The \-\-source option lets +you used a tarred set of /proc data saved earlier, possibly on a +different machine. The \-\-kernel and \-\-realmem options let you +specify a couple things that smem cannot discover on its own. + + +.TP +.BI "\-K " KERNEL ", \-\-kernel=" KERNEL +Path to an uncompressed kernel image. This lets smem include the size +of the kernel's code and statically allocated data in the systemwide +(\-w) output. (To obtain an uncompressed image of a kernel on disk, you +may need to build the kernel yourself, then locate file vmlinux in the +source tree.) + +.TP +.BI "\-R " REALMEM ", \-\-realmem=" REALMEM +Amount of physical RAM. This lets smem detect the amount of memory used +by firmware/hardware in the systemwide (\-w) output. If provided, it +will also be used as the total memory size to base percentages on. +Example: --realmem=1024M + +.TP +.BI "\-S " SOURCE ", \-\-source=" SOURCE +/proc data source. This lets you specify an alternate source of the +/proc data. For example, you can capture data from an embedded system +using smemcap, and parse the data later on a different machine. If the +\-\-source option is not included, smem reports memory usage on the +running system. + +.SS REPORT BY +If none of the following options are included, smem reports memory usage +by process. + +.TP +.B \-m, \-\-mappings +Report memory usage by mapping. + +.TP +.B \-u, \-\-users +Report memory usage by user. + +.TP +.B \-w, \-\-system +Report systemwide memory usage summary. + +.SS FILTER BY +If none of these options are included, memory usage is reported for all +processes, users, or mappings. (Note: If you are running as a non-root +user, and if you are not using the \-\-source options, then you will +only see data from processes whose /proc/ information you have access +to.) + +.TP +.BI "\-M " MAPFILTER ", \-\-mapfilter=" MAPFILTER +Mapping filter regular expression. + +.TP +.BI "\-P " PROCESSFILTER ", \-\-processfilter=" PROCESSFILTER +Process filter regular expression. + +.TP +.BI "\-U " USERFILTER ", \-\-userfilter=" USERFILTER +User filter regular expression. + +.SS OUTPUT FORMATTING + +.TP +.B \-a, \-\-autosize +Size columns to fit terminal size. + +.TP +.BI "\-c " COLUMNS ", \-\-columns=" COLUMNS +Columns to show. + +.TP +.B \-H, \-\-no\-header +Disable header line. + +.TP +.B \-k, \-\-abbreviate +Show unit suffixes. + +.TP +.B \-n, \-\-numeric +Show numeric user IDs instead of usernames. + +.TP +.B \-p, \-\-percent +Show percentages. + +.TP +.B \-r, \-\-reverse +Reverse sort. + +.TP +.BI "\-s " SORT ", \-\-sort=" SORT +Field to sort on. + +.TP +.B \-t, \-\-totals +Show totals. + +.SS OUTPUT TYPE +These options specify graphical output styles. + +.TP +.BI "\-\-bar=" BAR +Show bar graph. + +.TP +.BI "\-\-pie=" PIE +Show pie graph. + +.PP + +.SH REQUIREMENTS +\fBsmem\fP requires: + +.IP \(bu 3 +Linux kernel providing 'Pss' metric in /proc//smaps (generally +2.6.27 or newer). +.IP \(bu +Python 2.x (at least 2.4 or so). +.IP \(bu +The matplotlib library +(only if you want to generate graphical charts). + +.SH EMBEDDED USAGE +To capture memory statistics on resource\-constrained systems, the +the \fBsmem\fP source includes a utility named \fBsmemcap\fP. +\fBsmemcap\fP captures all /proc entries required by \fBsmem\fP +and outputs them as an uncompressed .tar file to STDOUT. +\fBsmem\fP can analyze the output using the \fB\-\-source\fP option. +\fBsmemcap\fP is small and does not require Python. +.PP +To use \fBsmemcap\fP: +.IP 1. 3 +Obtain the smem source at http://selenic.com/repo/smem +.IP 2. +Compile \fIsmemcap.c\fP for your target system. +.IP 3. +Run \fBsmemcap\fP on the target system and save the output: +.br +smemcap > memorycapture.tar +.IP 4. +Copy the output to another machine and run smem on it: +.br +smem -S memorycapture.tar + +.SH FILES +.I /proc/$pid/cmdline +.PP +.I /proc/$pid/smaps +.PP +.I /proc/$pid/stat +.PP +.I /proc/meminfo +.PP +.I /proc/version + +.SH RESOURCES +Main Web Site: http://www.selenic.com/smem + +Source code repository: http://selenic.com/repo/smem + +Mailing list: http://selenic.com/mailman/listinfo/smem + +.SH "SEE ALSO" +.BR free (1), +.BR pmap (1), +.BR proc (5), +.BR ps (1), +.BR top (1), +.BR vmstat (8) + +.SH COPYING +Copyright (C) 2008-2009 Matt Mackall. Free use of this software +is granted under the terms of the GNU General Public License +version 2 or later. + +.SH AUTHOR +\fBsmem\fP was written by Matt Mackall. + diff --git a/smemcap.c b/smemcap.c new file mode 100644 index 0000000..641c63b --- /dev/null +++ b/smemcap.c @@ -0,0 +1,114 @@ +/* + smem - a tool for meaningful memory reporting + + Copyright 2008-2009 Matt Mackall + + This software may be used and distributed according to the terms of + the GNU General Public License version 2 or later, incorporated + herein by reference. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +struct fileblock; +struct fileblock { + char data[512]; + struct fileblock *next; +}; + +int writeheader(int destfd, const char *path, int mode, int uid, int gid, + int size, int mtime, int type) +{ + char header[512]; + int i, sum; + + memset(header, 0, 512); + + sprintf(header, "%s", path); + sprintf(header + 100, "%07o", mode & 0777); + sprintf(header + 108, "%07o", uid); + sprintf(header + 116, "%07o", gid); + sprintf(header + 124, "%011o", size); + sprintf(header + 136, "%07o", mtime); + sprintf(header + 148, " %1d", type); + + /* fix checksum */ + for (i = sum = 0; i < 512; i++) + sum += header[i]; + sprintf(header + 148, "%06o", sum); + + return write(destfd, header, 512); +} + +int archivefile(const char *path, int destfd) +{ + struct fileblock *start, *cur; + struct fileblock **prev = &start; + int fd, r, size = 0; + struct stat s; + + /* buffer and stat the file */ + fd = open(path, O_RDONLY); + fstat(fd, &s); + + do { + cur = calloc(1, sizeof(struct fileblock)); + *prev = cur; + prev = &cur->next; + r = read(fd, cur->data, 512); + if (r > 0) + size += r; + } while (r == 512); + + close(fd); + + /* write archive header */ + writeheader(destfd, path, s.st_mode, s.st_uid, + s.st_gid, size, s.st_mtime, 0); + + /* dump file contents */ + for (cur = start; size > 0; size -= 512) { + write(destfd, cur->data, 512); + start = cur; + cur = cur->next; + free(start); + } +} + +int archivejoin(const char *sub, const char *name, int destfd) +{ + char path[256]; + sprintf(path, "%s/%s", sub, name); + return archivefile(path, destfd); +} + +int main(int argc, char *argv[]) +{ + DIR *d; + struct dirent *de; + struct stat s; + + chdir("/proc"); + archivefile("meminfo", 1); + archivefile("version", 1); + + d = opendir("."); + while ((de = readdir(d))) + if (de->d_name[0] >= '0' && de->d_name[0] <= '9') { + stat (de->d_name, &s); + writeheader(1, de->d_name, 0555, s.st_uid, + s.st_gid, 0, s.st_mtime, 5); + archivejoin(de->d_name, "smaps", 1); + archivejoin(de->d_name, "cmdline", 1); + archivejoin(de->d_name, "stat", 1); + } + + return 0; +}