Ticket #628: smaller-timefix-allow-user-controlled-metadata-and-dont-break-tests.patch.txt

diff -rN -u old-timestamps/docs/frontends/webapi.txt new-timestamps/docs/frontends/webapi.txt
--- old-timestamps/docs/frontends/webapi.txt    2009-04-08 21:17:06.000000000 -0600
+++ new-timestamps/docs/frontends/webapi.txt    2009-04-08 21:17:08.000000000 -0600
@@ -381,28 +381,44 @@
 GET /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=json
 
   This returns a machine-parseable JSON-encoded description of the given
-  object. The JSON always contains a list, and the first element of the list
-  is always a flag that indicates whether the referenced object is a file or a
-  directory. If it is a file, then the information includes file size and URI,
-  like this:
+  object. The JSON always contains a list, and the first element of the list is
+  always a flag that indicates whether the referenced object is a file or a
+  directory. If it is a capability to a file, then the information includes
+  file size and URI, like this:
 
    GET /uri/$FILECAP?t=json :
+
+    [ "filenode", { "ro_uri": file_uri,
+                    "verify_uri": verify_uri,
+                    "size": bytes,
+                    "mutable": false,
+                    } ]
+
+  If it is a capability to a directory followed by a path from that directory
+  to a file, then the information also includes metadata from the link to the
+  file in the parent directory, like this:
+
    GET /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=json :
 
     [ "filenode", { "ro_uri": file_uri,
                     "verify_uri": verify_uri,
                     "size": bytes,
                     "mutable": false,
-                    "metadata": {"ctime": 1202777696.7564139,
-                                 "mtime": 1202777696.7564139
+                    "metadata": {
+                                 "ctime": 1202777696.7564139,
+                                 "mtime": 1202777696.7564139,
+                                 "__sys": {
+                                         "linkcrtime": 1202777696.7564139,
+                                         "linkmotime": 1202777696.7564139,
                                  }
+                               }
                     } ]
 
   If it is a directory, then it includes information about the children of
   this directory, as a mapping from child name to a set of data about the
   child (the same data that would appear in a corresponding GET?t=json of the
   child itself). The child entries also include metadata about each child,
-  including creation- and modification- timestamps. The output looks like
+  including link-creation- and link-change- timestamps. The output looks like
   this:
 
    GET /uri/$DIRCAP?t=json :
@@ -418,13 +434,21 @@
                                                 "metadata": {
                                                   "ctime": 1202777696.7564139,
                                                   "mtime": 1202777696.7564139
+                                                  "__sys": {
+                                                    "linkcrtime": 1202777696.7564139,
+                                                    "linkmotime": 1202777696.7564139,
                                                  }
+                                                }
                                                } ],
                      "subdir":  [ "dirnode", { "rw_uri": rwuri,
                                                "ro_uri": rouri,
                                                 "metadata": {
                                                   "ctime": 1202778102.7589991,
                                                   "mtime": 1202778111.2160511,
+                                                  "__sys": {
+                                                    "linkcrtime": 1202777696.7564139,
+                                                    "linkmotime": 1202777696.7564139,
+                                                   }
                                                  }
                                               } ]
                     } } ]
diff -rN -u old-timestamps/docs/specifications/dirnodes.txt new-timestamps/docs/specifications/dirnodes.txt
--- old-timestamps/docs/specifications/dirnodes.txt     2009-04-08 21:17:06.000000000 -0600
+++ new-timestamps/docs/specifications/dirnodes.txt     2009-04-08 21:17:08.000000000 -0600
@@ -176,30 +176,29 @@
  netstring(cap) = 4+len(cap)
  encrypted(cap) = 16+cap+32
  JSON({}) = 2
- JSON({ctime=float,mtime=float}): 57
- netstring(metadata) = 4+57 = 61
+ JSON({ctime=float,mtime=float,'__sys':{linkcrtime=float,linkmotime=float}}): 137
+ netstring(metadata) = 4+137 = 141
 
 so a CHK entry is:
- 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+57
-And a 15-byte filename gives a 336-byte entry. When the entry points at a
+ 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+137
+And a 15-byte filename gives a 416-byte entry. When the entry points at a
 subdirectory instead of a file, the entry is a little bit smaller. So an
-empty directory uses 0 bytes, a directory with one child uses about 336
-bytes, a directory with two children uses about 672, etc.
+empty directory uses 0 bytes, a directory with one child uses about 416
+bytes, a directory with two children uses about 832, etc.
 
 When the dirnode data is encoding using our default 3-of-10, that means we
-get 112ish bytes of data in each share per child.
+get 139ish bytes of data in each share per child.
 
 The pubkey, signature, and hashes form the first 935ish bytes of the
 container, then comes our data, then about 1216 bytes of encprivkey. So if we
 read the first:
 
  1kB: we get 65bytes of dirnode data : only empty directories
- 1kiB: 89bytes of dirnode data : maybe one short-named subdir
- 2kB: 1065bytes: about 9 entries
- 3kB: 2065bytes: about 18 entries, or 7.5 entries plus the encprivkey
- 4kB: 3065bytes: about 27 entries, or about 16.5 plus the encprivkey
+ 2kB: 1065bytes: about 8
+ 3kB: 2065bytes: about 15 entries, or 6 entries plus the encprivkey
+ 4kB: 3065bytes: about 22 entries, or about 13 plus the encprivkey
 
-So we've written the code to do an initial read of 2kB from each share when
+So we've written the code to do an initial read of 4kB from each share when
 we read the mutable file, which should give good performance (one RTT) for
 small directories.
 
diff -rN -u old-timestamps/src/allmydata/dirnode.py new-timestamps/src/allmydata/dirnode.py
--- old-timestamps/src/allmydata/dirnode.py     2009-04-08 21:17:06.000000000 -0600
+++ new-timestamps/src/allmydata/dirnode.py     2009-04-08 21:17:08.000000000 -0600
@@ -83,15 +83,41 @@
                 metadata = children[name][1].copy()
             else:
                 metadata = {"ctime": now,
-                            "mtime": now}
-            if new_metadata is None:
-                # update timestamps
+                            "mtime": now,
+                            "__sys": {
+                                "linkcrtime": now,
+                                "linkmotime": now,
+                                }
+                            }
+
+            if new_metadata is not None:
+                # Overwrite all metadata.
+                newmd = new_metadata.copy()
+
+                # Except '__sys'.
+                if newmd.has_key('__sys'):
+                    del newmd['__sys']
+                if metadata.has_key('__sys'):
+                    newmd['__sys'] = metadata['__sys']
+
+                metadata = newmd
+            else:
+                # For backwards compatibility with Tahoe < 1.4.0:
                 if "ctime" not in metadata:
                     metadata["ctime"] = now
                 metadata["mtime"] = now
-            else:
-                # just replace it
-                metadata = new_metadata.copy()
+
+            # update timestamps
+            sysmd = metadata.get('__sys', {})
+            if not 'linkcrtime' in sysmd:
+                if "ctime" in metadata:
+                    # In Tahoe < 1.4.0 we used the word "ctime" to mean what Tahoe >= 1.4.0
+                    # calls "linkcrtime".
+                    sysmd["linkcrtime"] = metadata["ctime"]
+                else:
+                    sysmd["linkcrtime"] = now
+            sysmd["linkmotime"] = now
+
             children[name] = (child, metadata)
         new_contents = self.node._pack_contents(children)
         return new_contents
diff -rN -u old-timestamps/src/allmydata/mutable/servermap.py new-timestamps/src/allmydata/mutable/servermap.py
--- old-timestamps/src/allmydata/mutable/servermap.py   2009-04-08 21:17:07.000000000 -0600
+++ new-timestamps/src/allmydata/mutable/servermap.py   2009-04-08 21:17:08.000000000 -0600
@@ -374,7 +374,7 @@
         # fixed-size slots so we can retrieve less data. For now, we'll just
         # read 2000 bytes, which also happens to read enough actual data to
         # pre-fetch a 9-entry dirnode.
-        self._read_size = 2000
+        self._read_size = 4000
         if mode == MODE_CHECK:
             # we use unpack_prefix_and_signature, so we need 1k
             self._read_size = 1000
diff -rN -u old-timestamps/src/allmydata/scripts/tahoe_ls.py new-timestamps/src/allmydata/scripts/tahoe_ls.py
--- old-timestamps/src/allmydata/scripts/tahoe_ls.py    2009-04-08 21:17:07.000000000 -0600
+++ new-timestamps/src/allmydata/scripts/tahoe_ls.py    2009-04-08 21:17:08.000000000 -0600
@@ -65,8 +65,20 @@
         name = unicode(name)
         child = children[name]
         childtype = child[0]
-        ctime = child[1]["metadata"].get("ctime")
-        mtime = child[1]["metadata"].get("mtime")
+
+        # linkcrtime is not really what unix filesystems mean by "ctime", but
+        # it *is* apparently what many or even most unix programmers and users
+        # think that a unix filesystem means by "ctime"...
+        ctime = child[1].get("metadata", {}).get('__sys', {}).get("linkcrtime")
+        if not ctime:
+            ctime = child[1]["metadata"].get("ctime")
+
+        # linkmotime is not really what unix filesystems mean by "mtime",
+        # because linkmotime is a property of the link and mtime is a property
+        # of the file contents...
+        mtime = child[1].get("metadata", {}).get('__sys', {}).get("linkmotime")
+        if not mtime:
+            mtime = child[1]["metadata"].get("mtime")
         rw_uri = child[1].get("rw_uri")
         ro_uri = child[1].get("ro_uri")
         if ctime:
diff -rN -u old-timestamps/src/allmydata/test/test_dirnode.py new-timestamps/src/allmydata/test/test_dirnode.py
--- old-timestamps/src/allmydata/test/test_dirnode.py   2009-04-08 21:17:07.000000000 -0600
+++ new-timestamps/src/allmydata/test/test_dirnode.py   2009-04-08 21:17:08.000000000 -0600
@@ -416,7 +416,7 @@
             d.addCallback(lambda res: n.get_metadata_for(u"child"))
             d.addCallback(lambda metadata:
                           self.failUnlessEqual(sorted(metadata.keys()),
-                                               ["ctime", "mtime"]))
+                                               ["__sys", "ctime", "mtime"]))
 
             d.addCallback(lambda res:
                           self.shouldFail(NoSuchChildError, "gcamap-no",
diff -rN -u old-timestamps/src/allmydata/util/time_format.py new-timestamps/src/allmydata/util/time_format.py
--- old-timestamps/src/allmydata/util/time_format.py    2009-04-08 21:17:07.000000000 -0600
+++ new-timestamps/src/allmydata/util/time_format.py    2009-04-08 21:17:08.000000000 -0600
@@ -19,6 +19,11 @@
         now = t()
     return datetime.datetime.utcfromtimestamp(now).isoformat(sep)
 
+def iso_local(now=None, sep='_', t=time.time):
+    if now is None:
+        now = t()
+    return datetime.datetime.fromtimestamp(now).isoformat(sep)
+
 def iso_utc_time_to_seconds(isotime, _conversion_re=re.compile(r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})[T_ ](?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})(?P<subsecond>\.\d+)?")):
     """
     The inverse of iso_utc().
diff -rN -u old-timestamps/src/allmydata/web/directory.py new-timestamps/src/allmydata/web/directory.py
--- old-timestamps/src/allmydata/web/directory.py       2009-04-08 21:17:07.000000000 -0600
+++ new-timestamps/src/allmydata/web/directory.py       2009-04-08 21:17:08.000000000 -0600
@@ -13,7 +13,7 @@
 
 from foolscap.eventual import fireEventually
 
-from allmydata.util import base32
+from allmydata.util import base32, time_format
 from allmydata.uri import from_string_dirnode
 from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \
      ExistingChildError, NoSuchChildError
@@ -592,16 +592,25 @@
         ctx.fillSlots("rename", rename)
 
         times = []
-        TIME_FORMAT = "%H:%M:%S %d-%b-%Y"
-        if "ctime" in metadata:
-            ctime = time.strftime(TIME_FORMAT,
-                                  time.localtime(metadata["ctime"]))
-            times.append("c: " + ctime)
-        if "mtime" in metadata:
-            mtime = time.strftime(TIME_FORMAT,
-                                  time.localtime(metadata["mtime"]))
+        linkcrtime = metadata.get('__sys', {}).get("linkcrtime")
+        if linkcrtime is not None:
+            times.append("lcr: " + time_format.iso_local(linkcrtime))
+        else:
+            # For backwards-compatibility with links last modified by Tahoe < 1.4.0:
+            if "ctime" in metadata:
+                ctime = time_format.iso_local(metadata["ctime"])
+                times.append("c: " + ctime)
+        linkmotime = metadata.get('__sys', {}).get("linkmotime")
+        if linkmotime is not None:
             if times:
                 times.append(T.br())
+            times.append("lmo: " + time_format.iso_local(linkmotime))
+        else:
+            # For backwards-compatibility with links last modified by Tahoe < 1.4.0:
+            if "mtime" in metadata:
+                mtime = time_format.iso_local(metadata["mtime"])
+                if times:
+                    times.append(T.br())
                 times.append("m: " + mtime)
         ctx.fillSlots("times", times)