Ticket #628: patch.txt

File patch.txt, 17.0 KB (added by zooko, at 2009-04-08T02:45:45Z)
Line 
1diff -rN -u old-trunk/docs/frontends/webapi.txt new-trunk/docs/frontends/webapi.txt
2--- old-trunk/docs/frontends/webapi.txt 2009-04-07 20:44:50.000000000 -0600
3+++ new-trunk/docs/frontends/webapi.txt 2009-04-07 20:44:54.000000000 -0600
4@@ -393,8 +393,11 @@
5                     "verify_uri": verify_uri,
6                     "size": bytes,
7                     "mutable": false,
8-                    "metadata": {"ctime": 1202777696.7564139,
9-                                 "mtime": 1202777696.7564139
10+                    "metadata": {
11+                                 "ctime": 1202777696.7564139,
12+                                 "mtime": 1202777696.7564139,
13+                                 "lcrtime": 1202777696.7564139,
14+                                 "lmotime": 1202777696.7564139,
15                                  }
16                     } ]
17 
18@@ -402,7 +405,7 @@
19   this directory, as a mapping from child name to a set of data about the
20   child (the same data that would appear in a corresponding GET?t=json of the
21   child itself). The child entries also include metadata about each child,
22-  including creation- and modification- timestamps. The output looks like
23+  including link-creation- and link-change- timestamps. The output looks like
24   this:
25 
26    GET /uri/$DIRCAP?t=json :
27@@ -418,6 +421,8 @@
28                                                 "metadata": {
29                                                   "ctime": 1202777696.7564139,
30                                                   "mtime": 1202777696.7564139
31+                                                  "lcrtime": 1202777696.7564139,
32+                                                  "lmotime": 1202777696.7564139,
33                                                  }
34                                                } ],
35                      "subdir":  [ "dirnode", { "rw_uri": rwuri,
36@@ -425,6 +430,8 @@
37                                                 "metadata": {
38                                                   "ctime": 1202778102.7589991,
39                                                   "mtime": 1202778111.2160511,
40+                                                  "lcrtime": 1202777696.7564139,
41+                                                  "lmotime": 1202777696.7564139,
42                                                  }
43                                               } ]
44                     } } ]
45diff -rN -u old-trunk/docs/specifications/dirnodes.txt new-trunk/docs/specifications/dirnodes.txt
46--- old-trunk/docs/specifications/dirnodes.txt  2009-04-07 20:44:50.000000000 -0600
47+++ new-trunk/docs/specifications/dirnodes.txt  2009-04-07 20:44:54.000000000 -0600
48@@ -176,30 +176,30 @@
49  netstring(cap) = 4+len(cap)
50  encrypted(cap) = 16+cap+32
51  JSON({}) = 2
52- JSON({ctime=float,mtime=float}): 57
53- netstring(metadata) = 4+57 = 61
54+ JSON({ctime=float,mtime=float,lcrtime=float,lmotime=float}): 117
55+ netstring(metadata) = 4+117 = 121
56 
57 so a CHK entry is:
58- 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+57
59-And a 15-byte filename gives a 336-byte entry. When the entry points at a
60+ 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+117
61+And a 15-byte filename gives a 396-byte entry. When the entry points at a
62 subdirectory instead of a file, the entry is a little bit smaller. So an
63-empty directory uses 0 bytes, a directory with one child uses about 336
64-bytes, a directory with two children uses about 672, etc.
65+empty directory uses 0 bytes, a directory with one child uses about 396
66+bytes, a directory with two children uses about 792, etc.
67 
68 When the dirnode data is encoding using our default 3-of-10, that means we
69-get 112ish bytes of data in each share per child.
70+get 132ish bytes of data in each share per child.
71 
72 The pubkey, signature, and hashes form the first 935ish bytes of the
73 container, then comes our data, then about 1216 bytes of encprivkey. So if we
74 read the first:
75 
76  1kB: we get 65bytes of dirnode data : only empty directories
77- 1kiB: 89bytes of dirnode data : maybe one short-named subdir
78- 2kB: 1065bytes: about 9 entries
79- 3kB: 2065bytes: about 18 entries, or 7.5 entries plus the encprivkey
80- 4kB: 3065bytes: about 27 entries, or about 16.5 plus the encprivkey
81+ 1kiB: 89bytes of dirnode data : only empty directories
82+ 2kB: 1065bytes: about 2 or 3 entries
83+ 3kB: 2065bytes: about 5 entries, or 2 entries plus the encprivkey
84+ 4kB: 3065bytes: about 8 entries, or about 5 plus the encprivkey
85 
86-So we've written the code to do an initial read of 2kB from each share when
87+So we've written the code to do an initial read of 4kB from each share when
88 we read the mutable file, which should give good performance (one RTT) for
89 small directories.
90 
91diff -rN -u old-trunk/src/allmydata/dirnode.py new-trunk/src/allmydata/dirnode.py
92--- old-trunk/src/allmydata/dirnode.py  2009-04-07 20:44:53.000000000 -0600
93+++ new-trunk/src/allmydata/dirnode.py  2009-04-07 20:44:55.000000000 -0600
94@@ -83,15 +83,39 @@
95                 metadata = children[name][1].copy()
96             else:
97                 metadata = {"ctime": now,
98-                            "mtime": now}
99-            if new_metadata is None:
100-                # update timestamps
101-                if "ctime" not in metadata:
102-                    metadata["ctime"] = now
103-                metadata["mtime"] = now
104-            else:
105-                # just replace it
106-                metadata = new_metadata.copy()
107+                            "mtime": now,
108+                            "lcrtime": now,
109+                            "lmotime": now,
110+                            }
111+
112+            if new_metadata is not None:
113+                # Overwrite all metadata.
114+                newmd = new_metadata.copy()
115+
116+                # Except that callers are not allowed to mess with the
117+                # link-timestamp metadata.
118+                for special in ['ctime', 'mtime', 'lcrtime', 'lmotime']:
119+                    if newmd.has_key(special):
120+                        del newmd[special]
121+                    if metadata.has_key(special):
122+                        newmd[special] = metadata[special]
123+
124+                metadata = newmd
125+
126+            # update timestamps
127+            if "lcrtime" not in metadata:
128+                if "ctime" in metadata:
129+                    # In Tahoe < 1.4.0 we used the word "ctime" to mean what Tahoe >= 1.4.0 calls "lcrtime".
130+                    metadata["lcrtime"] = metadata["ctime"]
131+                else:
132+                    metadata["lmotime"] = now
133+            metadata["lmotime"] = now
134+
135+            # For backwards compatibility with Tahoe < 1.4.0:
136+            if "ctime" not in metadata:
137+                metadata["ctime"] = now
138+            metadata["mtime"] = now
139+
140             children[name] = (child, metadata)
141         new_contents = self.node._pack_contents(children)
142         return new_contents
143diff -rN -u old-trunk/src/allmydata/mutable/servermap.py new-trunk/src/allmydata/mutable/servermap.py
144--- old-trunk/src/allmydata/mutable/servermap.py        2009-04-07 20:44:53.000000000 -0600
145+++ new-trunk/src/allmydata/mutable/servermap.py        2009-04-07 20:44:55.000000000 -0600
146@@ -374,7 +374,7 @@
147         # fixed-size slots so we can retrieve less data. For now, we'll just
148         # read 2000 bytes, which also happens to read enough actual data to
149         # pre-fetch a 9-entry dirnode.
150-        self._read_size = 2000
151+        self._read_size = 4000
152         if mode == MODE_CHECK:
153             # we use unpack_prefix_and_signature, so we need 1k
154             self._read_size = 1000
155diff -rN -u old-trunk/src/allmydata/scripts/tahoe_backup.py new-trunk/src/allmydata/scripts/tahoe_backup.py
156--- old-trunk/src/allmydata/scripts/tahoe_backup.py     2009-04-07 20:44:53.000000000 -0600
157+++ new-trunk/src/allmydata/scripts/tahoe_backup.py     2009-04-07 20:44:55.000000000 -0600
158@@ -4,6 +4,7 @@
159 import urllib
160 import simplejson
161 import datetime
162+import platform
163 from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS
164 from allmydata.scripts.common_http import do_http
165 from allmydata import uri
166@@ -69,11 +69,7 @@
167     metadata = {}
168 
169     # posix stat(2) metadata, depends on the platform
170-    os.stat_float_times(True)
171     s = os.stat(path)
172-    metadata["ctime"] = s.st_ctime
173-    metadata["mtime"] = s.st_mtime
174-
175     misc_fields = ("st_mode", "st_ino", "st_dev", "st_uid", "st_gid")
176     macos_misc_fields = ("st_rsize", "st_creator", "st_type")
177     for field in misc_fields + macos_misc_fields:
178diff -rN -u old-trunk/src/allmydata/scripts/tahoe_ls.py new-trunk/src/allmydata/scripts/tahoe_ls.py
179--- old-trunk/src/allmydata/scripts/tahoe_ls.py 2009-04-07 20:44:53.000000000 -0600
180+++ new-trunk/src/allmydata/scripts/tahoe_ls.py 2009-04-07 20:44:55.000000000 -0600
181@@ -65,8 +65,20 @@
182         name = unicode(name)
183         child = children[name]
184         childtype = child[0]
185-        ctime = child[1]["metadata"].get("ctime")
186-        mtime = child[1]["metadata"].get("mtime")
187+
188+        # lcrtime is not really what unix filesystems mean by "ctime", but it
189+        # *is* apparently what many or even most unix programmers and users
190+        # think that a unix filesystem means by "ctime"...
191+        ctime = child[1]["metadata"].get("lcrtime")
192+        if not ctime:
193+            ctime = child[1]["metadata"].get("ctime")
194+
195+        # lmotime is not really what unix filesystems mean by "mtime", because
196+        # lmotime is a property of the link and mtime is a property of the file
197+        # contents...
198+        mtime = child[1]["metadata"].get("lmotime")
199+        if not mtime:
200+            mtime = child[1]["metadata"].get("mtime")
201         rw_uri = child[1].get("rw_uri")
202         ro_uri = child[1].get("ro_uri")
203         if ctime:
204diff -rN -u old-trunk/src/allmydata/test/test_cli.py new-trunk/src/allmydata/test/test_cli.py
205--- old-trunk/src/allmydata/test/test_cli.py    2009-04-07 20:44:53.000000000 -0600
206+++ new-trunk/src/allmydata/test/test_cli.py    2009-04-07 20:44:55.000000000 -0600
207@@ -949,7 +949,7 @@
208                 self.failUnlessEqual(fu, 0)
209                 self.failUnlessEqual(fr, 3)
210                 # empty, home, home/parent, home/parent/subdir
211-                self.failUnlessEqual(dc, 0)
212+                self.failUnlessEqual(dc, 0, out)
213                 self.failUnlessEqual(dr, 4)
214         d.addCallback(_check4a)
215 
216diff -rN -u old-trunk/src/allmydata/test/test_dirnode.py new-trunk/src/allmydata/test/test_dirnode.py
217--- old-trunk/src/allmydata/test/test_dirnode.py        2009-04-07 20:44:53.000000000 -0600
218+++ new-trunk/src/allmydata/test/test_dirnode.py        2009-04-07 20:44:55.000000000 -0600
219@@ -416,7 +416,7 @@
220             d.addCallback(lambda res: n.get_metadata_for(u"child"))
221             d.addCallback(lambda metadata:
222                           self.failUnlessEqual(sorted(metadata.keys()),
223-                                               ["ctime", "mtime"]))
224+                                               ["ctime", "lcrtime", "lmotime", "mtime"]))
225 
226             d.addCallback(lambda res:
227                           self.shouldFail(NoSuchChildError, "gcamap-no",
228@@ -439,7 +439,7 @@
229                 self.failUnlessEqual(child.get_uri(),
230                                      fake_file_uri.to_string())
231                 self.failUnlessEqual(sorted(metadata.keys()),
232-                                     ["ctime", "mtime"])
233+                                     ["ctime", "lcrtime", "lmotime", "mtime"])
234             d.addCallback(_check_child_and_metadata2)
235 
236             d.addCallback(lambda res:
237@@ -448,36 +448,41 @@
238                 child, metadata = res
239                 self.failUnless(isinstance(child, FakeDirectoryNode))
240                 self.failUnlessEqual(sorted(metadata.keys()),
241-                                     ["ctime", "mtime"])
242+                                     ["ctime", "lcrtime", "lmotime", "mtime"])
243             d.addCallback(_check_child_and_metadata3)
244 
245             # set_uri + metadata
246-            # it should be possible to add a child without any metadata
247+            # it should not be possible to add a child without any metadata
248             d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri.to_string(), {}))
249             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
250-            d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
251+            def _has_ltimes(metadata):
252+                self.failUnless(metadata.has_key('ctime'))
253+                self.failUnless(metadata.has_key('mtime'))
254+                self.failUnless(metadata.has_key('lcrtime'))
255+                self.failUnless(metadata.has_key('lmotime'))
256+            d.addCallback(_has_ltimes)
257+
258+            # nor to override the link timestamps with the "metadata" argument
259+            d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri.to_string(), {'lcrtime': "bogus"}))
260+            d.addCallback(lambda res: n.get_metadata_for(u"c2"))
261+            def _has_good_lcrtime(metadata):
262+                self.failUnless(metadata.has_key('lcrtime'))
263+                self.failIfEqual(metadata['lcrtime'], 'bogus')
264+            d.addCallback(_has_good_lcrtime)
265 
266             # if we don't set any defaults, the child should get timestamps
267             d.addCallback(lambda res: n.set_uri(u"c3", fake_file_uri.to_string()))
268             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
269             d.addCallback(lambda metadata:
270                           self.failUnlessEqual(sorted(metadata.keys()),
271-                                               ["ctime", "mtime"]))
272-
273-            # or we can add specific metadata at set_uri() time, which
274-            # overrides the timestamps
275-            d.addCallback(lambda res: n.set_uri(u"c4", fake_file_uri.to_string(),
276-                                                {"key": "value"}))
277-            d.addCallback(lambda res: n.get_metadata_for(u"c4"))
278-            d.addCallback(lambda metadata:
279-                          self.failUnlessEqual(metadata, {"key": "value"}))
280+                                               ["ctime", "lcrtime", "lmotime", "mtime"]))
281 
282             d.addCallback(lambda res: n.delete(u"c2"))
283             d.addCallback(lambda res: n.delete(u"c3"))
284             d.addCallback(lambda res: n.delete(u"c4"))
285 
286             # set_node + metadata
287-            # it should be possible to add a child without any metadata
288+            # it should be impossible to add a child without any metadata
289             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
290             d.addCallback(lambda res: self.client.create_empty_dirnode())
291             d.addCallback(lambda n2:
292diff -rN -u old-trunk/src/allmydata/util/time_format.py new-trunk/src/allmydata/util/time_format.py
293--- old-trunk/src/allmydata/util/time_format.py 2009-04-07 20:44:53.000000000 -0600
294+++ new-trunk/src/allmydata/util/time_format.py 2009-04-07 20:44:55.000000000 -0600
295@@ -19,6 +19,11 @@
296         now = t()
297     return datetime.datetime.utcfromtimestamp(now).isoformat(sep)
298 
299+def iso_local(now=None, sep='_', t=time.time):
300+    if now is None:
301+        now = t()
302+    return datetime.datetime.fromtimestamp(now).isoformat(sep)
303+
304 def iso_utc_time_to_seconds(isotime, _conversion_re=re.compile(r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})[T_ ](?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})(?P<subsecond>\.\d+)?")):
305     """
306     The inverse of iso_utc().
307diff -rN -u old-trunk/src/allmydata/web/directory.py new-trunk/src/allmydata/web/directory.py
308--- old-trunk/src/allmydata/web/directory.py    2009-04-07 20:44:53.000000000 -0600
309+++ new-trunk/src/allmydata/web/directory.py    2009-04-07 20:44:55.000000000 -0600
310@@ -13,7 +13,7 @@
311 
312 from foolscap.eventual import fireEventually
313 
314-from allmydata.util import base32
315+from allmydata.util import base32, time_format
316 from allmydata.uri import from_string_dirnode
317 from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \
318      ExistingChildError, NoSuchChildError
319@@ -592,16 +592,25 @@
320         ctx.fillSlots("rename", rename)
321 
322         times = []
323-        TIME_FORMAT = "%H:%M:%S %d-%b-%Y"
324-        if "ctime" in metadata:
325-            ctime = time.strftime(TIME_FORMAT,
326-                                  time.localtime(metadata["ctime"]))
327-            times.append("c: " + ctime)
328-        if "mtime" in metadata:
329-            mtime = time.strftime(TIME_FORMAT,
330-                                  time.localtime(metadata["mtime"]))
331+        lcrtime = metadata.get("lcrtime")
332+        if lcrtime is not None:
333+            times.append("lcr: " + time_format.iso_local(lcrtime))
334+        else:
335+            # For backwards-compatibility with links last modified by Tahoe < 1.4.0:
336+            if "ctime" in metadata:
337+                ctime = time_format.iso_local(metadata["ctime"])
338+                times.append("c: " + ctime)
339+        lmotime = metadata.get("lmotime")
340+        if lmotime is not None:
341             if times:
342                 times.append(T.br())
343+            times.append("lmo: " + time_format.iso_local(lmotime))
344+        else:
345+            # For backwards-compatibility with links last modified by Tahoe < 1.4.0:
346+            if "mtime" in metadata:
347+                mtime = time_format.iso_local(metadata["mtime"])
348+                if times:
349+                    times.append(T.br())
350                 times.append("m: " + mtime)
351         ctx.fillSlots("times", times)
352