Added tag 1.5 for changeset 98273ce331bb

If catching the exception is done with "except Exception as e:" this
will work only from 2.6. To maintain backward compatibility with 2.4 I
have used sys.exc_info(). A bit of a wart, but it works.

COPYING

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

capture

@@ -1,12 +0,0 @@
-#!/bin/sh
-# example of capturing target data for smem
-
-# capture a memory data snapshot with realtime priority
-mkdir -p $1
-chrt --fifo 99 \
-  cp -a --parents /proc/[0-9]*/{smaps,cmdline,stat} /proc/meminfo $1
-
-# build a compressed tarball of snapshot
-cd $1/proc
-tar czf ../../$1.tgz *
-

smem

@@ -1,16 +1,27 @@
-#!/usr/bin/python
-import re, os, sys, pwd, grp, optparse, errno, tarfile
+#!/usr/bin/env python
+#
+# smem - a tool for meaningful memory reporting
+#
+# Copyright 2008-2009 Matt Mackall <mpm@selenic.com>
+#
+# This software may be used and distributed according to the terms of
+# the GNU General Public License version 2 or later, incorporated
+# herein by reference.
 
-_ucache = {}
-_gcache = {}
+import re, os, sys, pwd, optparse, errno, tarfile
+
+warned = False
 
 class procdata(object):
     def __init__(self, source):
+        self._ucache = {}
+        self._gcache = {}
         self.source = source and source or ""
+        self._memdata = None
     def _list(self):
         return os.listdir(self.source + "/proc")
     def _read(self, f):
-        return file(self.source + '/proc/' + f).read()
+        return open(self.source + '/proc/' + f).read()
     def _readlines(self, f):
         return self._read(f).splitlines(True)
     def _stat(self, f):
@@ -23,69 +34,124 @@ class procdata(object):
     def mapdata(self, pid):
         return self._readlines('%s/smaps' % pid)
     def memdata(self):
-        return self._readlines('meminfo')
+        if self._memdata is None:
+            self._memdata = self._readlines('meminfo')
+        return self._memdata
+    def version(self):
+        return self._readlines('version')[0]
     def pidname(self, pid):
-        l = self._read('%d/stat' % pid)
-        return l[l.find('(') + 1: l.find(')')]
+        try:
+            l = self._read('%d/stat' % pid)
+            return l[l.find('(') + 1: l.find(')')]
+        except:
+            return '?'
     def pidcmd(self, pid):
-        c = self._read('%s/cmdline' % pid)[:-1]
-        return c.replace('\0', ' ')
+        try:
+            c = self._read('%s/cmdline' % pid)[:-1]
+            return c.replace('\0', ' ')
+        except:
+            return '?'
     def piduser(self, pid):
-        return self._stat('%d/cmdline' % pid).st_uid
+        try:
+            return self._stat('%d' % pid).st_uid
+        except:
+            return -1
     def pidgroup(self, pid):
-        return self._stat('%d/cmdline' % pid).st_gid
+        try:
+            return self._stat('%d' % pid).st_gid
+        except:
+            return -1
     def username(self, uid):
-        if uid not in _ucache:
-            _ucache[uid] = pwd.getpwuid(uid)[0]
-        return _ucache[uid]
+        if uid == -1:
+            return '?'
+        if uid not in self._ucache:
+            try:
+                self._ucache[uid] = pwd.getpwuid(uid)[0]
+            except KeyError:
+                self._ucache[uid] = str(uid)
+        return self._ucache[uid]
     def groupname(self, gid):
-        if gid not in _gcache:
-            _gcache[gid] = pwd.getgrgid(gid)[0]
-        return _gcache[gid]
+        if gid == -1:
+            return '?'
+        if gid not in self._gcache:
+            try:
+                self._gcache[gid] = pwd.getgrgid(gid)[0]
+            except KeyError:
+                self._gcache[gid] = str(gid)
+        return self._gcache[gid]
 
 class tardata(procdata):
     def __init__(self, source):
-        self.source = source
+        procdata.__init__(self, source)
         self.tar = tarfile.open(source)
     def _list(self):
         for ti in self.tar:
-            if ti.name.endswith('/'):
-                yield ti.name[:-1]
+            if ti.name.endswith('/smaps'):
+                d,f = ti.name.split('/')
+                yield d
     def _read(self, f):
         return self.tar.extractfile(f).read()
     def _readlines(self, f):
         return self.tar.extractfile(f).readlines()
     def piduser(self, p):
-        return self.tar.getmember("%s/cmdline" % p).uid
+        t = self.tar.getmember("%d" % p)
+        if t.uname:
+            self._ucache[t.uid] = t.uname
+        return t.uid
     def pidgroup(self, p):
-        return self.tar.getmember("%s/cmdline" % p).gid
+        t = self.tar.getmember("%d" % p)
+        if t.gname:
+            self._gcache[t.gid] = t.gname
+        return t.gid
+    def username(self, u):
+        return self._ucache.get(u, str(u))
+    def groupname(self, g):
+        return self._gcache.get(g, str(g))
 
 _totalmem = 0
 def totalmem():
     global _totalmem
     if not _totalmem:
         if options.realmem:
-            _totalmem = fromunits(options.realmem)
+            _totalmem = fromunits(options.realmem) / 1024
         else:
             _totalmem = memory()['memtotal']
-    return _totalmem / 1024
+    return _totalmem
 
 _kernelsize = 0
 def kernelsize():
     global _kernelsize
     if not _kernelsize and options.kernel:
-        d = os.popen("size %s" % options.kernel).readlines()[1]
-        _kernelsize = int(d.split()[3]) / 1024
+        try:
+            d = os.popen("size %s" % options.kernel).readlines()[1]
+            _kernelsize = int(d.split()[3]) / 1024
+        except:
+            try:
+                # try some heuristic to find gzipped part in kernel image
+                packedkernel = open(options.kernel).read()
+                pos = packedkernel.find('\x1F\x8B')
+                if pos >= 0 and pos < 25000:
+                    sys.stderr.write("Maybe uncompressed kernel can be extracted by the command:\n"
+                            "  dd if=%s bs=1 skip=%d | gzip -d >%s.unpacked\n\n" % (options.kernel, pos, options.kernel))
+            except:
+                pass
+            sys.stderr.write("Parameter '%s' should be an original uncompressed compiled kernel file.\n\n" % options.kernel)
     return _kernelsize
 
 def pidmaps(pid):
+    global warned
     maps = {}
     start = None
+    seen = False
+    empty = True
     for l in src.mapdata(pid):
-    	f = l.split()
-	if f[-1] == 'kB':
+        empty = False
+        f = l.split()
+        if f[-1] == 'kB':
+            if f[0].startswith('Pss'):
+                seen = True
             maps[start][f[0][:-1].lower()] = int(f[1])
-        else:
+        elif '-' in f[0] and ':' not in f[0]: # looks like a mapping range
             start, end = f[0].split('-')
             start = int(start, 16)
             name = "<anonymous>"
@@ -95,10 +161,16 @@ def pidmaps(pid):
                                offset=int(f[2], 16),
                                device=f[3], inode=f[4], name=name)
 
+    if not empty and not seen and not warned:
+        sys.stderr.write('warning: kernel does not appear to support PSS measurement\n')
+        warned = True
+        if not options.sort:
+            options.sort = 'rss'
+
     if options.mapfilter:
         f = {}
         for m in maps:
-            if not filter(options.mapfilter, m, lambda x: maps[x]['name']):
+            if not filters(options.mapfilter, m, lambda x: maps[x]['name']):
                 f[m] = maps[m]
         return f
 
@@ -127,7 +199,7 @@ def units(x):
     s = ''
     if x == 0:
         return '0'
-    for s in ('', 'K', 'M', 'G'):
+    for s in ('', 'K', 'M', 'G', 'T'):
         if x < 1024:
             break
         x /= 1024.0
@@ -135,34 +207,27 @@ def units(x):
 
 def fromunits(x):
     s = dict(k=2**10, K=2**10, kB=2**10, KB=2**10,
-             M=2**20, MB=2**20, G=2**30, GB=2**30)
+             M=2**20, MB=2**20, G=2**30, GB=2**30,
+             T=2**40, TB=2**40)
     for k,v in s.items():
         if x.endswith(k):
-            return int(float(x[:len(k)])*v)
-
-_ucache = {}
-def username(uid):
-    if uid not in _ucache:
-        _ucache[uid] = pwd.getpwuid(uid)[0]
-    return _ucache[uid]
+            return int(float(x[:-len(k)])*v)
+    sys.stderr.write("Memory size should be written with units, for example 1024M\n")
+    sys.exit(-1)
 
 def pidusername(pid):
-    return username(src.piduser(pid))
+    return src.username(src.piduser(pid))
 
-_gcache = {}
-def groupname(gid):
-    if gid not in _gcache:
-        _gcache[gid] = grp.getgrgid(gid)[0]
-    return _gcache[gid]
-
-def showamount(a):
+def showamount(a, total):
     if options.abbreviate:
         return units(a * 1024)
     elif options.percent:
-        return "%.2f%%" % (100.0 * a / totalmem())
+        if total == 0:
+            return 'N/A'
+        return "%.2f%%" % (100.0 * a / total)
     return a
 
-def filter(opt, arg, *sources):
+def filters(opt, arg, *sources):
     if not opt:
         return False
 
@@ -175,7 +240,7 @@ def pidtotals(pid):
     maps = pidmaps(pid)
     t = dict(size=0, rss=0, pss=0, shared_clean=0, shared_dirty=0,
              private_clean=0, private_dirty=0, referenced=0, swap=0)
-    for m in maps.iterkeys():
+    for m in maps:
         for k in t:
             t[k] += maps[m].get(k, 0)
 
@@ -187,8 +252,8 @@ def pidtotals(pid):
 def processtotals(pids):
     totals = {}
     for pid in pids:
-        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
-                filter(options.userfilter, pid, pidusername)):
+        if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or
+                filters(options.userfilter, pid, pidusername)):
             continue
         try:
             p = pidtotals(pid)
@@ -204,11 +269,11 @@ def showpids():
 
     def showuser(p):
         if options.numeric:
-            return piduser(p)
+            return src.piduser(p)
         return pidusername(p)
 
     fields = dict(
-        pid=('PID', lambda n: n, '% 5s', lambda x: len(p),
+        pid=('PID', lambda n: n, '% 5s', lambda x: len(pt),
              'process ID'),
         user=('User', showuser, '%-8s', lambda x: len(dict.fromkeys(x)),
               'owner of process'),
@@ -236,13 +301,13 @@ def showpids():
 def maptotals(pids):
     totals = {}
     for pid in pids:
-        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
-                filter(options.userfilter, pid, pidusername)):
+        if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or
+                filters(options.userfilter, pid, pidusername)):
             continue
         try:
             maps = pidmaps(pid)
             seen = {}
-            for m in maps.iterkeys():
+            for m in maps:
                 name = maps[m]['name']
                 if name not in totals:
                     t = dict(size=0, rss=0, pss=0, shared_clean=0,
@@ -258,8 +323,8 @@ def maptotals(pids):
                     t['pids'] += 1
                     seen[name] = 1
                 totals[name] = t
-        except:
-            raise
+        except EnvironmentError:
+            continue
     return totals
 
 def showmaps():
@@ -301,15 +366,15 @@ def showmaps():
 def usertotals(pids):
     totals = {}
     for pid in pids:
-        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
-                filter(options.userfilter, pid, pidusername)):
+        if (filters(options.processfilter, pid, src.pidname, src.pidcmd) or
+                filters(options.userfilter, pid, pidusername)):
             continue
         try:
             maps = pidmaps(pid)
             if len(maps) == 0:
                 continue
-        except:
-            raise
+        except EnvironmentError:
+            continue
         user = src.piduser(pid)
         if user not in totals:
             t = dict(size=0, rss=0, pss=0, shared_clean=0,
@@ -318,7 +383,7 @@ def usertotals(pids):
         else:
             t = totals[user]
 
-        for m in maps.iterkeys():
+        for m in maps:
             for k in t:
                 t[k] += maps[m].get(k, 0)
 
@@ -333,7 +398,7 @@ def showusers():
     def showuser(u):
         if options.numeric:
             return u
-        return username(u)
+        return src.username(u)
 
     fields = dict(
         user=('User', showuser, '%-8s', None,
@@ -358,26 +423,34 @@ def showusers():
 
 def showsystem():
     t = totalmem()
-    k = kernelsize()
+    ki = kernelsize()
     m = memory()
+
     mt = m['memtotal']
-    fh = t - mt - k
     f = m['memfree']
+
+    # total amount used by hardware
+    fh = max(t - mt - ki, 0)
+
+    # total amount mapped into userspace (ie mapped an unmapped pages)
     u = m['anonpages'] + m['mapped']
+
+    # total amount allocated by kernel not for userspace
     kd = mt - f - u
-    kdd = (m['buffers'] + m['sreclaimable'] +
-           (m['cached'] - m['anonpages']))
+
+    # total amount in kernel caches
+    kdc = m['buffers'] + m['sreclaimable'] + (m['cached'] - m['mapped'])
 
     l = [("firmware/hardware", fh, 0),
-         ("kernel image", k, 0),
-         ("kernel dynamic memory", kd, kdd),
+         ("kernel image", ki, 0),
+         ("kernel dynamic memory", kd, kdc),
          ("userspace memory", u, m['mapped']),
          ("free memory", f, f)]
 
     fields = dict(
-        order=('Order', lambda n: n, '% 1s', lambda x: len(p),
+        order=('Order', lambda n: n, '% 1s', lambda x: '',
              'hierarchical order'),
-        area=('Area', lambda n: l[n][0], '%-24s', lambda x: len(l),
+        area=('Area', lambda n: l[n][0], '%-24s', lambda x: '',
              'memory area'),
         used=('Used', lambda n: l[n][1], '%10a', sum,
               'area in use'),
@@ -390,11 +463,44 @@ def showsystem():
     showtable(range(len(l)), fields, columns.split(), options.sort or 'order')
 
 def showfields(fields, f):
-    if f != list:
-        print "unknown field", f
-    print "known fields:"
-    for l in sorted(fields.keys()):
-        print "%-8s %s" % (l, fields[l][-1])
+    if type(f) in (list, set):
+        print("unknown fields: " + " ".join(f))
+    else:
+        print("unknown field %s" % f)
+    print("known fields:")
+    for l in sorted(fields):
+        print("%-8s %s" % (l, fields[l][-1]))
+
+def autosize(columns, fields, rows):
+    colsizes = {}
+    for c in columns:
+        sizes = [1]
+
+        if not options.no_header:
+            sizes.append(len(fields[c][0]))
+
+        if (options.abbreviate or options.percent) and 'a' in fields[c][2]:
+            sizes.append(7)
+        else:
+            for r in rows:
+                sizes.append(len(str(fields[c][1](r))))
+
+        colsizes[c] = max(sizes)
+
+    overflowcols = set(["command", "map"]) & set(columns)
+    if len(overflowcols) > 0:
+        overflowcol = overflowcols.pop()
+        totnoflow = sum(colsizes.values()) - colsizes[overflowcol]
+        try:
+            ttyrows, ttycolumns = os.popen('stty size', 'r').read().split()
+            ttyrows, ttycolumns = int(ttyrows), int(ttycolumns)
+        except:
+            ttyrows, ttycolumns = (24, 80)
+        maxflowcol = ttycolumns - totnoflow - len(columns)
+        maxflowcol = max(maxflowcol, 10)
+        colsizes[overflowcol] = min(colsizes[overflowcol], maxflowcol)
+
+    return colsizes
 
 def showtable(rows, fields, columns, sort):
     header = ""
@@ -410,17 +516,31 @@ def showtable(rows, fields, columns, sort):
     if options.bar:
         columns.append(options.bar)
 
-    for n in columns:
-        if n not in fields:
-            showfields(fields, n)
-            sys.exit(-1)
+    mt = totalmem()
+    st = memory()['swaptotal']
 
+    missing = set(columns) - set(fields)
+    if len(missing) > 0:
+        showfields(fields, missing)
+        sys.exit(-1)
+
+    if options.autosize:
+        colsizes = autosize(columns, fields, rows)
+    else:
+        colsizes = {}
+
+    for n in columns:
         f = fields[n][2]
         if 'a' in f:
-            formatter.append(showamount)
+            if n == 'swap':
+                formatter.append(lambda x: showamount(x, st))
+            else:
+                formatter.append(lambda x: showamount(x, mt))
             f = f.replace('a', 's')
         else:
             formatter.append(lambda x: x)
+        if n in colsizes:
+            f = re.sub(r"[0-9]+", str(colsizes[n]), f)
         format += f + " "
         header += f % fields[n][0] + " "
 
@@ -439,10 +559,10 @@ def showtable(rows, fields, columns, sort):
         return
 
     if not options.no_header:
-        print header
+        print(header)
 
     for k,r in l:
-        print format % tuple([f(v) for f,v in zip(formatter, r)])
+        print(format % tuple([f(v) for f,v in zip(formatter, r)]))
 
     if options.totals:
         # totals
@@ -454,8 +574,8 @@ def showtable(rows, fields, columns, sort):
             else:
                 t.append("")
 
-        print "-" * len(header)
-        print format % tuple([f(v) for f,v in zip(formatter, t)])
+        print("-" * len(header))
+        print(format % tuple([f(v) for f,v in zip(formatter, t)]))
 
 def showpie(l, sort):
     try:
@@ -474,15 +594,12 @@ def showpie(l, sort):
     s = sum(values)
     unused = tm - s
     t = 0
-    c = 0
-    while values and (t + values[-1 - c] < (tm * .02) or
-                      values[-1 - c] < (tm * .005)):
-        c += 1
+    while values and (t + values[-1] < (tm * .02) or
+                      values[-1] < (tm * .005)):
         t += values.pop()
         labels.pop()
-    if c > 1:
-        values = values[:-c]
-        labels = labels[:-c]
+
+    if t:
         values.append(t)
         labels.append('other')
 
@@ -539,6 +656,7 @@ def showbar(l, columns, sort):
     pylab.legend([p[0] for p in pl], key)
     pylab.show()
 
+
 parser = optparse.OptionParser("%prog [options]")
 parser.add_option("-H", "--no-header", action="store_true",
                   help="disable header line")
@@ -546,6 +664,8 @@ parser.add_option("-c", "--columns", type="str",
                   help="columns to show")
 parser.add_option("-t", "--totals", action="store_true",
                   help="show totals")
+parser.add_option("-a", "--autosize", action="store_true",
+                  help="size columns to fit terminal size")
 
 parser.add_option("-R", "--realmem", type="str",
                   help="amount of physical RAM")
@@ -605,7 +725,8 @@ try:
         showsystem()
     else:
         showpids()
-except IOError, e:
+except IOError:
+    _, e, _ = sys.exc_info()
     if e.errno == errno.EPIPE:
         pass
 except KeyboardInterrupt:

smem.8

@@ -0,0 +1,214 @@
+.TH SMEM 8 "03/15/2010" "" ""
+
+.SH NAME
+smem \- Report memory usage with shared memory divided proportionally.
+
+.SH SYNOPSIS
+.B smem
+.RI [ options ]
+
+.SH DESCRIPTION
+\fBsmem\fP reports physical memory usage, taking shared memory pages
+into account.  Unshared memory is reported as the USS (Unique Set Size).
+Shared memory is divided evenly among the processes sharing that memory.
+The unshared memory (USS) plus a process's proportion of shared memory
+is reported as the PSS (Proportional Set Size).  The USS and PSS only
+include physical memory usage.  They do not include memory that has been
+swapped out to disk.
+
+Memory can be reported by process, by user, by mapping, or systemwide.
+Both text mode and graphical output are available.
+
+.SH OPTIONS
+
+.SS GENERAL OPTIONS
+
+.TP
+.B \-h, \-\-help
+Show help.
+
+.SS SOURCE DATA
+By default, smem will pull most of the data it needs from the /proc
+filesystem of the system it is running on.  The \-\-source option lets
+you used a tarred set of /proc data saved earlier, possibly on a
+different machine.  The \-\-kernel and \-\-realmem options let you
+specify a couple things that smem cannot discover on its own.
+
+
+.TP
+.BI "\-K " KERNEL ", \-\-kernel=" KERNEL
+Path to an uncompressed kernel image.  This lets smem include the size
+of the kernel's code and statically allocated data in the systemwide
+(\-w) output.  (To obtain an uncompressed image of a kernel on disk, you
+may need to build the kernel yourself, then locate file vmlinux in the
+source tree.)
+
+.TP
+.BI "\-R " REALMEM ", \-\-realmem=" REALMEM
+Amount of physical RAM.  This lets smem detect the amount of memory used
+by firmware/hardware in the systemwide (\-w) output.  If provided, it
+will also be used as the total memory size to base percentages on.
+Example: --realmem=1024M
+
+.TP
+.BI "\-S " SOURCE ", \-\-source=" SOURCE
+/proc data source.  This lets you specify an alternate source of the
+/proc data.  For example, you can capture data from an embedded system
+using smemcap, and parse the data later on a different machine.  If the
+\-\-source option is not included, smem reports memory usage on the
+running system.
+
+.SS REPORT BY
+If none of the following options are included, smem reports memory usage
+by process.
+
+.TP
+.B \-m, \-\-mappings
+Report memory usage by mapping.
+
+.TP
+.B \-u, \-\-users
+Report memory usage by user.
+
+.TP
+.B \-w, \-\-system
+Report systemwide memory usage summary.
+
+.SS FILTER BY
+If none of these options are included, memory usage is reported for all
+processes, users, or mappings.  (Note: If you are running as a non-root
+user, and if you are not using the \-\-source options, then you will
+only see data from processes whose /proc/ information you have access
+to.)
+
+.TP
+.BI "\-M " MAPFILTER ", \-\-mapfilter=" MAPFILTER
+Mapping filter regular expression.
+
+.TP
+.BI "\-P " PROCESSFILTER ", \-\-processfilter=" PROCESSFILTER
+Process filter regular expression.
+
+.TP
+.BI "\-U " USERFILTER ", \-\-userfilter=" USERFILTER
+User filter regular expression.
+
+.SS OUTPUT FORMATTING
+
+.TP
+.B \-a, \-\-autosize
+Size columns to fit terminal size.
+
+.TP
+.BI "\-c " COLUMNS ", \-\-columns=" COLUMNS
+Columns to show.
+
+.TP
+.B \-H, \-\-no\-header
+Disable header line.
+
+.TP
+.B \-k, \-\-abbreviate
+Show unit suffixes.
+
+.TP
+.B \-n, \-\-numeric
+Show numeric user IDs instead of usernames.
+
+.TP
+.B \-p, \-\-percent
+Show percentages.
+
+.TP
+.B \-r, \-\-reverse
+Reverse sort.
+
+.TP
+.BI "\-s " SORT ", \-\-sort=" SORT
+Field to sort on.
+
+.TP
+.B \-t, \-\-totals
+Show totals.
+
+.SS OUTPUT TYPE
+These options specify graphical output styles.
+
+.TP
+.BI "\-\-bar=" BAR
+Show bar graph.
+
+.TP
+.BI "\-\-pie=" PIE
+Show pie graph.
+
+.PP
+
+.SH REQUIREMENTS
+\fBsmem\fP requires:
+
+.IP \(bu 3
+Linux kernel providing 'Pss' metric in /proc/<pid>/smaps (generally
+2.6.27 or newer).
+.IP \(bu
+Python 2.x (at least 2.4 or so).
+.IP \(bu
+The matplotlib library
+(only if you want to generate graphical charts).
+
+.SH EMBEDDED USAGE
+To capture memory statistics on resource\-constrained systems, the
+the \fBsmem\fP source includes a utility named \fBsmemcap\fP.
+\fBsmemcap\fP captures all /proc entries required by \fBsmem\fP
+and outputs them as an uncompressed .tar file to STDOUT.
+\fBsmem\fP can analyze the output using the \fB\-\-source\fP option.
+\fBsmemcap\fP is small and does not require Python.
+.PP
+To use \fBsmemcap\fP:
+.IP 1. 3
+Obtain the smem source at http://selenic.com/repo/smem
+.IP 2.
+Compile \fIsmemcap.c\fP for your target system.
+.IP 3.
+Run \fBsmemcap\fP on the target system and save the output:
+.br
+smemcap > memorycapture.tar
+.IP 4.
+Copy the output to another machine and run smem on it:
+.br
+smem -S memorycapture.tar
+
+.SH FILES
+.I /proc/$pid/cmdline
+.PP
+.I /proc/$pid/smaps
+.PP
+.I /proc/$pid/stat
+.PP
+.I /proc/meminfo
+.PP
+.I /proc/version
+
+.SH RESOURCES
+Main Web Site: http://www.selenic.com/smem
+
+Source code repository: http://selenic.com/repo/smem
+
+Mailing list: http://selenic.com/mailman/listinfo/smem
+
+.SH "SEE ALSO"
+.BR free (1),
+.BR pmap (1),
+.BR proc (5),
+.BR ps (1),
+.BR top (1),
+.BR vmstat (8)
+
+.SH COPYING
+Copyright (C) 2008-2009 Matt Mackall.  Free use of this software
+is granted under the terms of the GNU General Public License
+version 2 or later.
+
+.SH AUTHOR
+\fBsmem\fP was written by Matt Mackall.
+

smemcap.c

@@ -0,0 +1,114 @@
+/*
+ smem - a tool for meaningful memory reporting
+
+ Copyright 2008-2009 Matt Mackall <mpm@selenic.com>
+
+ This software may be used and distributed according to the terms of
+ the GNU General Public License version 2 or later, incorporated
+ herein by reference.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <memory.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <dirent.h>
+
+struct fileblock;
+struct fileblock {
+	char data[512];
+	struct fileblock *next;
+};
+
+int writeheader(int destfd, const char *path, int mode, int uid, int gid,
+		int size, int mtime, int type)
+{
+	char header[512];
+	int i, sum;
+
+	memset(header, 0, 512);
+
+	sprintf(header, "%s", path);
+	sprintf(header + 100, "%07o", mode & 0777);
+	sprintf(header + 108, "%07o", uid);
+	sprintf(header + 116, "%07o", gid);
+	sprintf(header + 124, "%011o", size);
+	sprintf(header + 136, "%07o", mtime);
+	sprintf(header + 148, "        %1d", type);
+
+	/* fix checksum */
+	for (i = sum = 0; i < 512; i++)
+               sum += header[i];
+	sprintf(header + 148, "%06o", sum);
+
+	return write(destfd, header, 512);
+}
+
+int archivefile(const char *path, int destfd)
+{
+	struct fileblock *start, *cur;
+	struct fileblock **prev = &start;
+	int fd, r, size = 0;
+	struct stat s;
+
+	/* buffer and stat the file */
+	fd = open(path, O_RDONLY);
+	fstat(fd, &s);
+
+	do {
+		cur = calloc(1, sizeof(struct fileblock));
+		*prev = cur;
+		prev = &cur->next;
+		r = read(fd, cur->data, 512);
+		if (r > 0)
+			size += r;
+	} while (r == 512);
+
+	close(fd);
+
+	/* write archive header */
+	writeheader(destfd, path, s.st_mode, s.st_uid,
+		    s.st_gid, size, s.st_mtime, 0);
+
+	/* dump file contents */
+	for (cur = start; size > 0; size -= 512) {
+		write(destfd, cur->data, 512);
+		start = cur;
+		cur = cur->next;
+		free(start);
+	}
+}
+
+int archivejoin(const char *sub, const char *name, int destfd)
+{
+	char path[256];
+	sprintf(path, "%s/%s", sub, name);
+	return archivefile(path, destfd);
+}
+
+int main(int argc, char *argv[])
+{
+	DIR *d;
+	struct dirent *de;
+	struct stat s;
+
+	chdir("/proc");
+	archivefile("meminfo", 1);
+	archivefile("version", 1);
+
+	d = opendir(".");
+	while ((de = readdir(d)))
+		if (de->d_name[0] >= '0' && de->d_name[0] <= '9') {
+			stat (de->d_name, &s);
+			writeheader(1, de->d_name, 0555, s.st_uid,
+				    s.st_gid, 0, s.st_mtime, 5);
+			archivejoin(de->d_name, "smaps", 1);
+			archivejoin(de->d_name, "cmdline", 1);
+			archivejoin(de->d_name, "stat", 1);
+		}
+
+	return 0;
+}