Context Navigation

source: trunk/src/allmydata/test/test_dirnode.py

Visit:

Last change on this file was fec97256, checked in by Alexandre Detiste <alexandre.detiste@…>, at 2025-01-06T21:51:37Z
trim Python2 syntax
Property mode set to `100644`
File size: 101.6 KB

Line
1	"""Tests for the dirnode module.
2
3	Ported to Python 3.
4	"""
5
6	import time
7	import unicodedata
8	from zope.interface import implementer
9	from twisted.trial import unittest
10	from twisted.internet import defer
11	from twisted.internet.interfaces import IConsumer
12	from twisted.python.filepath import FilePath
13	from allmydata import uri, dirnode
14	from allmydata.client import _Client
15	from allmydata.crypto.rsa import create_signing_keypair
16	from allmydata.immutable import upload
17	from allmydata.immutable.literal import LiteralFileNode
18	from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
19	ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
20	MustBeDeepImmutableError, MustBeReadonlyError, \
21	IDeepCheckResults, IDeepCheckAndRepairResults, \
22	MDMF_VERSION, SDMF_VERSION
23	from allmydata.mutable.filenode import MutableFileNode
24	from allmydata.mutable.common import (
25	UncoordinatedWriteError,
26	derive_mutable_keys,
27	)
28	from allmydata.util import hashutil, base32
29	from allmydata.util.netstring import split_netstring
30	from allmydata.monitor import Monitor
31	from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
32	ErrorMixin
33	from allmydata.test.mutable.util import (
34	FakeStorage,
35	make_nodemaker_with_peers,
36	make_peer,
37	)
38	from allmydata.test.no_network import GridTestMixin
39	from allmydata.unknown import UnknownNode, strip_prefix_for_ro
40	from allmydata.nodemaker import NodeMaker
41	from base64 import b32decode
42	from cryptography.hazmat.primitives.serialization import load_pem_private_key
43	import allmydata.test.common_util as testutil
44
45	from hypothesis import given
46	from hypothesis.strategies import text
47
48
49	@implementer(IConsumer)
50	class MemAccum:
51	def registerProducer(self, producer, streaming):
52	self.producer = producer
53	self.producer.resumeProducing()
54	pass
55	def unregisterProducer(self):
56	pass
57	def write(self, data):
58	assert not hasattr(self, 'data')
59	self.data = data
60	self.producer.resumeProducing()
61
62	setup_py_uri = b"URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
63	one_uri = b"URI:LIT:n5xgk" # LIT for "one"
64	mut_write_uri = b"URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
65	mdmf_write_uri = b"URI:MDMF:x533rhbm6kiehzl5kj3s44n5ie:4gif5rhneyd763ouo5qjrgnsoa3bg43xycy4robj2rf3tvmhdl3a"
66	empty_litdir_uri = b"URI:DIR2-LIT:"
67	tiny_litdir_uri = b"URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
68	mut_read_uri = b"URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
69	mdmf_read_uri = b"URI:MDMF-RO:d4cydxselputycfzkw6qgz4zv4:4gif5rhneyd763ouo5qjrgnsoa3bg43xycy4robj2rf3tvmhdl3a"
70	future_write_uri = b"x-tahoe-crazy://I_am_from_the_future."
71	future_read_uri = b"x-tahoe-crazy-readonly://I_am_from_the_future."
72	future_nonascii_write_uri = u"x-tahoe-even-more-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
73	future_nonascii_read_uri = u"x-tahoe-even-more-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
74
75	# 'o' 'n' 'e-macron'
76	one_nfc = u"on\u0113"
77	one_nfd = u"one\u0304"
78
79	class Dirnode(GridTestMixin, unittest.TestCase,
80	testutil.ReallyEqualMixin, testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
81
82	def _do_create_test(self, mdmf=False):
83	c = self.g.clients[0]
84
85	self.expected_manifest = []
86	self.expected_verifycaps = set()
87	self.expected_storage_indexes = set()
88
89	d = None
90	if mdmf:
91	d = c.create_dirnode(version=MDMF_VERSION)
92	else:
93	d = c.create_dirnode()
94	def _then(n):
95	# /
96	self.rootnode = n
97	backing_node = n._node
98	if mdmf:
99	self.failUnlessEqual(backing_node.get_version(),
100	MDMF_VERSION)
101	else:
102	self.failUnlessEqual(backing_node.get_version(),
103	SDMF_VERSION)
104	self.failUnless(n.is_mutable())
105	u = n.get_uri()
106	self.failUnless(u)
107	cap_formats = []
108	if mdmf:
109	cap_formats = [b"URI:DIR2-MDMF:",
110	b"URI:DIR2-MDMF-RO:",
111	b"URI:DIR2-MDMF-Verifier:"]
112	else:
113	cap_formats = [b"URI:DIR2:",
114	b"URI:DIR2-RO",
115	b"URI:DIR2-Verifier:"]
116	rw, ro, v = cap_formats
117	self.failUnless(u.startswith(rw), u)
118	u_ro = n.get_readonly_uri()
119	self.failUnless(u_ro.startswith(ro), u_ro)
120	u_v = n.get_verify_cap().to_string()
121	self.failUnless(u_v.startswith(v), u_v)
122	u_r = n.get_repair_cap().to_string()
123	self.failUnlessReallyEqual(u_r, u)
124	self.expected_manifest.append( ((), u) )
125	self.expected_verifycaps.add(u_v)
126	si = n.get_storage_index()
127	self.expected_storage_indexes.add(base32.b2a(si))
128	expected_si = n._uri.get_storage_index()
129	self.failUnlessReallyEqual(si, expected_si)
130
131	d = n.list()
132	d.addCallback(lambda res: self.failUnlessEqual(res, {}))
133	d.addCallback(lambda res: n.has_child(u"missing"))
134	d.addCallback(lambda res: self.failIf(res))
135
136	fake_file_uri = make_mutable_file_uri()
137	other_file_uri = make_mutable_file_uri()
138	m = c.nodemaker.create_from_cap(fake_file_uri)
139	ffu_v = m.get_verify_cap().to_string()
140	self.expected_manifest.append( ((u"child",) , m.get_uri()) )
141	self.expected_verifycaps.add(ffu_v)
142	self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
143	d.addCallback(lambda res: n.set_uri(u"child",
144	fake_file_uri, fake_file_uri))
145	d.addCallback(lambda res:
146	self.shouldFail(ExistingChildError, "set_uri-no",
147	"child 'child' already exists",
148	n.set_uri, u"child",
149	other_file_uri, other_file_uri,
150	overwrite=False))
151	# /
152	# /child = mutable
153
154	d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
155
156	# /
157	# /child = mutable
158	# /subdir = directory
159	def _created(subdir):
160	self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
161	self.subdir = subdir
162	new_v = subdir.get_verify_cap().to_string()
163	assert isinstance(new_v, bytes)
164	self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
165	self.expected_verifycaps.add(new_v)
166	si = subdir.get_storage_index()
167	self.expected_storage_indexes.add(base32.b2a(si))
168	d.addCallback(_created)
169
170	d.addCallback(lambda res:
171	self.shouldFail(ExistingChildError, "mkdir-no",
172	"child 'subdir' already exists",
173	n.create_subdirectory, u"subdir",
174	overwrite=False))
175
176	d.addCallback(lambda res: n.list())
177	d.addCallback(lambda children:
178	self.failUnlessReallyEqual(set(children.keys()),
179	set([u"child", u"subdir"])))
180
181	d.addCallback(lambda res: n.start_deep_stats().when_done())
182	def _check_deepstats(stats):
183	self.failUnless(isinstance(stats, dict))
184	expected = {"count-immutable-files": 0,
185	"count-mutable-files": 1,
186	"count-literal-files": 0,
187	"count-files": 1,
188	"count-directories": 2,
189	"size-immutable-files": 0,
190	"size-literal-files": 0,
191	#"size-directories": 616, # varies
192	#"largest-directory": 616,
193	"largest-directory-children": 2,
194	"largest-immutable-file": 0,
195	}
196	for k,v in expected.items():
197	self.failUnlessReallyEqual(stats[k], v,
198	"stats[%s] was %s, not %s" %
199	(k, stats[k], v))
200	self.failUnless(stats["size-directories"] > 500,
201	stats["size-directories"])
202	self.failUnless(stats["largest-directory"] > 500,
203	stats["largest-directory"])
204	self.failUnlessReallyEqual(stats["size-files-histogram"], [])
205	d.addCallback(_check_deepstats)
206
207	d.addCallback(lambda res: n.build_manifest().when_done())
208	def _check_manifest(res):
209	manifest = res["manifest"]
210	self.failUnlessReallyEqual(sorted(manifest),
211	sorted(self.expected_manifest))
212	stats = res["stats"]
213	_check_deepstats(stats)
214	self.failUnlessReallyEqual(self.expected_verifycaps,
215	res["verifycaps"])
216	self.failUnlessReallyEqual(self.expected_storage_indexes,
217	res["storage-index"])
218	d.addCallback(_check_manifest)
219
220	def _add_subsubdir(res):
221	return self.subdir.create_subdirectory(u"subsubdir")
222	d.addCallback(_add_subsubdir)
223	# /
224	# /child = mutable
225	# /subdir = directory
226	# /subdir/subsubdir = directory
227	d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
228	d.addCallback(lambda subsubdir:
229	self.failUnless(isinstance(subsubdir,
230	dirnode.DirectoryNode)))
231	d.addCallback(lambda res: n.get_child_at_path(u""))
232	d.addCallback(lambda res: self.failUnlessReallyEqual(res.get_uri(),
233	n.get_uri()))
234
235	d.addCallback(lambda res: n.get_metadata_for(u"child"))
236	d.addCallback(lambda metadata:
237	self.failUnlessEqual(set(metadata.keys()),
238	set(["tahoe"])))
239
240	d.addCallback(lambda res:
241	self.shouldFail(NoSuchChildError, "gcamap-no",
242	"nope",
243	n.get_child_and_metadata_at_path,
244	u"subdir/nope"))
245	d.addCallback(lambda res:
246	n.get_child_and_metadata_at_path(u""))
247	def _check_child_and_metadata1(res):
248	child, metadata = res
249	self.failUnless(isinstance(child, dirnode.DirectoryNode))
250	# edge-metadata needs at least one path segment
251	self.failUnlessEqual(set(metadata.keys()), set([]))
252	d.addCallback(_check_child_and_metadata1)
253	d.addCallback(lambda res:
254	n.get_child_and_metadata_at_path(u"child"))
255
256	def _check_child_and_metadata2(res):
257	child, metadata = res
258	self.failUnlessReallyEqual(child.get_uri(),
259	fake_file_uri)
260	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
261	d.addCallback(_check_child_and_metadata2)
262
263	d.addCallback(lambda res:
264	n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
265	def _check_child_and_metadata3(res):
266	child, metadata = res
267	self.failUnless(isinstance(child, dirnode.DirectoryNode))
268	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
269	d.addCallback(_check_child_and_metadata3)
270
271	# set_uri + metadata
272	# it should be possible to add a child without any metadata
273	d.addCallback(lambda res: n.set_uri(u"c2",
274	fake_file_uri, fake_file_uri,
275	{}))
276	d.addCallback(lambda res: n.get_metadata_for(u"c2"))
277	d.addCallback(lambda metadata:
278	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
279
280	# You can't override the link timestamps.
281	d.addCallback(lambda res: n.set_uri(u"c2",
282	fake_file_uri, fake_file_uri,
283	{ 'tahoe': {'linkcrtime': "bogus"}}))
284	d.addCallback(lambda res: n.get_metadata_for(u"c2"))
285	def _has_good_linkcrtime(metadata):
286	self.failUnless('tahoe' in metadata)
287	self.failUnless('linkcrtime' in metadata['tahoe'])
288	self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
289	d.addCallback(_has_good_linkcrtime)
290
291	# if we don't set any defaults, the child should get timestamps
292	d.addCallback(lambda res: n.set_uri(u"c3",
293	fake_file_uri, fake_file_uri))
294	d.addCallback(lambda res: n.get_metadata_for(u"c3"))
295	d.addCallback(lambda metadata:
296	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
297
298	# we can also add specific metadata at set_uri() time
299	d.addCallback(lambda res: n.set_uri(u"c4",
300	fake_file_uri, fake_file_uri,
301	{"key": "value"}))
302	d.addCallback(lambda res: n.get_metadata_for(u"c4"))
303	d.addCallback(lambda metadata:
304	self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
305	(metadata['key'] == "value"), metadata))
306
307	d.addCallback(lambda res: n.delete(u"c2"))
308	d.addCallback(lambda res: n.delete(u"c3"))
309	d.addCallback(lambda res: n.delete(u"c4"))
310
311	# set_node + metadata
312	# it should be possible to add a child without any metadata except for timestamps
313	d.addCallback(lambda res: n.set_node(u"d2", n, {}))
314	d.addCallback(lambda res: c.create_dirnode())
315	d.addCallback(lambda n2:
316	self.shouldFail(ExistingChildError, "set_node-no",
317	"child 'd2' already exists",
318	n.set_node, u"d2", n2,
319	overwrite=False))
320	d.addCallback(lambda res: n.get_metadata_for(u"d2"))
321	d.addCallback(lambda metadata:
322	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
323
324	# if we don't set any defaults, the child should get timestamps
325	d.addCallback(lambda res: n.set_node(u"d3", n))
326	d.addCallback(lambda res: n.get_metadata_for(u"d3"))
327	d.addCallback(lambda metadata:
328	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
329
330	# we can also add specific metadata at set_node() time
331	d.addCallback(lambda res: n.set_node(u"d4", n,
332	{"key": "value"}))
333	d.addCallback(lambda res: n.get_metadata_for(u"d4"))
334	d.addCallback(lambda metadata:
335	self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
336	(metadata["key"] == "value"), metadata))
337
338	d.addCallback(lambda res: n.delete(u"d2"))
339	d.addCallback(lambda res: n.delete(u"d3"))
340	d.addCallback(lambda res: n.delete(u"d4"))
341
342	# metadata through set_children()
343	d.addCallback(lambda res:
344	n.set_children({
345	u"e1": (fake_file_uri, fake_file_uri),
346	u"e2": (fake_file_uri, fake_file_uri, {}),
347	u"e3": (fake_file_uri, fake_file_uri,
348	{"key": "value"}),
349	}))
350	d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
351	d.addCallback(lambda res:
352	self.shouldFail(ExistingChildError, "set_children-no",
353	"child 'e1' already exists",
354	n.set_children,
355	{ u"e1": (other_file_uri,
356	other_file_uri),
357	u"new": (other_file_uri,
358	other_file_uri),
359	},
360	overwrite=False))
361	# and 'new' should not have been created
362	d.addCallback(lambda res: n.list())
363	d.addCallback(lambda children: self.failIf(u"new" in children))
364	d.addCallback(lambda res: n.get_metadata_for(u"e1"))
365	d.addCallback(lambda metadata:
366	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
367	d.addCallback(lambda res: n.get_metadata_for(u"e2"))
368	d.addCallback(lambda metadata:
369	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
370	d.addCallback(lambda res: n.get_metadata_for(u"e3"))
371	d.addCallback(lambda metadata:
372	self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
373	(metadata["key"] == "value"), metadata))
374
375	d.addCallback(lambda res: n.delete(u"e1"))
376	d.addCallback(lambda res: n.delete(u"e2"))
377	d.addCallback(lambda res: n.delete(u"e3"))
378
379	# metadata through set_nodes()
380	d.addCallback(lambda res:
381	n.set_nodes({ u"f1": (n, None),
382	u"f2": (n, {}),
383	u"f3": (n, {"key": "value"}),
384	}))
385	d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
386	d.addCallback(lambda res:
387	self.shouldFail(ExistingChildError, "set_nodes-no",
388	"child 'f1' already exists",
389	n.set_nodes, { u"f1": (n, None),
390	u"new": (n, None), },
391	overwrite=False))
392	# and 'new' should not have been created
393	d.addCallback(lambda res: n.list())
394	d.addCallback(lambda children: self.failIf(u"new" in children))
395	d.addCallback(lambda res: n.get_metadata_for(u"f1"))
396	d.addCallback(lambda metadata:
397	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
398	d.addCallback(lambda res: n.get_metadata_for(u"f2"))
399	d.addCallback(lambda metadata:
400	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
401	d.addCallback(lambda res: n.get_metadata_for(u"f3"))
402	d.addCallback(lambda metadata:
403	self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
404	(metadata["key"] == "value"), metadata))
405
406	d.addCallback(lambda res: n.delete(u"f1"))
407	d.addCallback(lambda res: n.delete(u"f2"))
408	d.addCallback(lambda res: n.delete(u"f3"))
409
410
411	d.addCallback(lambda res:
412	n.set_metadata_for(u"child",
413	{"tags": ["web2.0-compatible"], "tahoe": {"bad": "mojo"}}))
414	d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
415	d.addCallback(lambda metadata:
416	self.failUnless((set(metadata.keys()) == set(["tags", "tahoe"])) and
417	metadata["tags"] == ["web2.0-compatible"] and
418	"bad" not in metadata["tahoe"], metadata))
419
420	d.addCallback(lambda res:
421	self.shouldFail(NoSuchChildError, "set_metadata_for-nosuch", "",
422	n.set_metadata_for, u"nosuch", {}))
423
424
425	def _start(res):
426	self._start_timestamp = time.time()
427	d.addCallback(_start)
428	# a long time ago, we used simplejson-1.7.1 (as shipped on Ubuntu
429	# 'gutsy'), which had a bug/misbehavior in which it would round
430	# all floats to hundredeths (it used str(num) instead of
431	# repr(num)). To prevent this bug from causing the test to fail,
432	# we stall for more than a few hundrededths of a second here.
433	# simplejson-1.7.3 does not have this bug, and anyways we've
434	# moved on to stdlib "json" which doesn't have it either.
435	d.addCallback(self.stall, 0.1)
436	d.addCallback(lambda res: n.add_file(u"timestamps",
437	upload.Data(b"stamp me", convergence=b"some convergence string")))
438	d.addCallback(self.stall, 0.1)
439	def _stop(res):
440	self._stop_timestamp = time.time()
441	d.addCallback(_stop)
442
443	d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
444	def _check_timestamp1(metadata):
445	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
446	tahoe_md = metadata["tahoe"]
447	self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
448
449	self.failUnlessGreaterOrEqualThan(tahoe_md["linkcrtime"],
450	self._start_timestamp)
451	self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
452	tahoe_md["linkcrtime"])
453	self.failUnlessGreaterOrEqualThan(tahoe_md["linkmotime"],
454	self._start_timestamp)
455	self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
456	tahoe_md["linkmotime"])
457	# Our current timestamp rules say that replacing an existing
458	# child should preserve the 'linkcrtime' but update the
459	# 'linkmotime'
460	self._old_linkcrtime = tahoe_md["linkcrtime"]
461	self._old_linkmotime = tahoe_md["linkmotime"]
462	d.addCallback(_check_timestamp1)
463	d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
464	d.addCallback(lambda res: n.set_node(u"timestamps", n))
465	d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
466	def _check_timestamp2(metadata):
467	self.failUnlessIn("tahoe", metadata)
468	tahoe_md = metadata["tahoe"]
469	self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
470
471	self.failUnlessReallyEqual(tahoe_md["linkcrtime"], self._old_linkcrtime)
472	self.failUnlessGreaterThan(tahoe_md["linkmotime"], self._old_linkmotime)
473	return n.delete(u"timestamps")
474	d.addCallback(_check_timestamp2)
475
476	d.addCallback(lambda res: n.delete(u"subdir"))
477	d.addCallback(lambda old_child:
478	self.failUnlessReallyEqual(old_child.get_uri(),
479	self.subdir.get_uri()))
480
481	d.addCallback(lambda res: n.list())
482	d.addCallback(lambda children:
483	self.failUnlessReallyEqual(set(children.keys()),
484	set([u"child"])))
485
486	uploadable1 = upload.Data(b"some data", convergence=b"converge")
487	d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
488	d.addCallback(lambda newnode:
489	self.failUnless(IImmutableFileNode.providedBy(newnode)))
490	uploadable2 = upload.Data(b"some data", convergence=b"stuff")
491	d.addCallback(lambda res:
492	self.shouldFail(ExistingChildError, "add_file-no",
493	"child 'newfile' already exists",
494	n.add_file, u"newfile",
495	uploadable2,
496	overwrite=False))
497	d.addCallback(lambda res: n.list())
498	d.addCallback(lambda children:
499	self.failUnlessReallyEqual(set(children.keys()),
500	set([u"child", u"newfile"])))
501	d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
502	d.addCallback(lambda metadata:
503	self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
504
505	uploadable3 = upload.Data(b"some data", convergence=b"converge")
506	d.addCallback(lambda res: n.add_file(u"newfile-metadata",
507	uploadable3,
508	{"key": "value"}))
509	d.addCallback(lambda newnode:
510	self.failUnless(IImmutableFileNode.providedBy(newnode)))
511	d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
512	d.addCallback(lambda metadata:
513	self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
514	(metadata['key'] == "value"), metadata))
515	d.addCallback(lambda res: n.delete(u"newfile-metadata"))
516
517	d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
518	def _created2(subdir2):
519	self.subdir2 = subdir2
520	# put something in the way, to make sure it gets overwritten
521	return subdir2.add_file(u"child", upload.Data(b"overwrite me",
522	b"converge"))
523	d.addCallback(_created2)
524
525	d.addCallback(lambda res:
526	n.move_child_to(u"child", self.subdir2))
527	d.addCallback(lambda res: n.list())
528	d.addCallback(lambda children:
529	self.failUnlessReallyEqual(set(children.keys()),
530	set([u"newfile", u"subdir2"])))
531	d.addCallback(lambda res: self.subdir2.list())
532	d.addCallback(lambda children:
533	self.failUnlessReallyEqual(set(children.keys()),
534	set([u"child"])))
535	d.addCallback(lambda res: self.subdir2.get(u"child"))
536	d.addCallback(lambda child:
537	self.failUnlessReallyEqual(child.get_uri(),
538	fake_file_uri))
539
540	# move it back, using new_child_name=
541	d.addCallback(lambda res:
542	self.subdir2.move_child_to(u"child", n, u"newchild"))
543	d.addCallback(lambda res: n.list())
544	d.addCallback(lambda children:
545	self.failUnlessReallyEqual(set(children.keys()),
546	set([u"newchild", u"newfile",
547	u"subdir2"])))
548	d.addCallback(lambda res: self.subdir2.list())
549	d.addCallback(lambda children:
550	self.failUnlessReallyEqual(set(children.keys()), set([])))
551
552	# now make sure that we honor overwrite=False
553	d.addCallback(lambda res:
554	self.subdir2.set_uri(u"newchild",
555	other_file_uri, other_file_uri))
556
557	d.addCallback(lambda res:
558	self.shouldFail(ExistingChildError, "move_child_to-no",
559	"child 'newchild' already exists",
560	n.move_child_to, u"newchild",
561	self.subdir2,
562	overwrite=False))
563	d.addCallback(lambda res: self.subdir2.get(u"newchild"))
564	d.addCallback(lambda child:
565	self.failUnlessReallyEqual(child.get_uri(),
566	other_file_uri))
567
568
569	# Setting the no-write field should diminish a mutable cap to read-only
570	# (for both files and directories).
571
572	d.addCallback(lambda ign: n.set_uri(u"mutable", other_file_uri, other_file_uri))
573	d.addCallback(lambda ign: n.get(u"mutable"))
574	d.addCallback(lambda mutable: self.failIf(mutable.is_readonly(), mutable))
575	d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
576	d.addCallback(lambda ign: n.get(u"mutable"))
577	d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
578	d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
579	d.addCallback(lambda ign: n.get(u"mutable"))
580	d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
581
582	d.addCallback(lambda ign: n.get(u"subdir2"))
583	d.addCallback(lambda subdir2: self.failIf(subdir2.is_readonly()))
584	d.addCallback(lambda ign: n.set_metadata_for(u"subdir2", {"no-write": True}))
585	d.addCallback(lambda ign: n.get(u"subdir2"))
586	d.addCallback(lambda subdir2: self.failUnless(subdir2.is_readonly(), subdir2))
587
588	d.addCallback(lambda ign: n.set_uri(u"mutable_ro", other_file_uri, other_file_uri,
589	metadata={"no-write": True}))
590	d.addCallback(lambda ign: n.get(u"mutable_ro"))
591	d.addCallback(lambda mutable_ro: self.failUnless(mutable_ro.is_readonly(), mutable_ro))
592
593	d.addCallback(lambda ign: n.create_subdirectory(u"subdir_ro", metadata={"no-write": True}))
594	d.addCallback(lambda ign: n.get(u"subdir_ro"))
595	d.addCallback(lambda subdir_ro: self.failUnless(subdir_ro.is_readonly(), subdir_ro))
596
597	return d
598
599	d.addCallback(_then)
600
601	d.addErrback(self.explain_error)
602	return d
603
604
605	def _do_initial_children_test(self, mdmf=False):
606	c = self.g.clients[0]
607	nm = c.nodemaker
608
609	kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
610	u"two": (nm.create_from_cap(setup_py_uri),
611	{"metakey": "metavalue"}),
612	u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
613	u"mdmf": (nm.create_from_cap(mdmf_write_uri, mdmf_read_uri), {}),
614	u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
615	u"fro": (nm.create_from_cap(None, future_read_uri), {}),
616	u"fut-unic": (nm.create_from_cap(future_nonascii_write_uri, future_nonascii_read_uri), {}),
617	u"fro-unic": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
618	u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
619	u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
620	}
621	if mdmf:
622	d = c.create_dirnode(kids, version=MDMF_VERSION)
623	else:
624	d = c.create_dirnode(kids)
625
626	def _created(dn):
627	self.failUnless(isinstance(dn, dirnode.DirectoryNode))
628	backing_node = dn._node
629	if mdmf:
630	self.failUnlessEqual(backing_node.get_version(),
631	MDMF_VERSION)
632	else:
633	self.failUnlessEqual(backing_node.get_version(),
634	SDMF_VERSION)
635	self.failUnless(dn.is_mutable())
636	self.failIf(dn.is_readonly())
637	self.failIf(dn.is_unknown())
638	self.failIf(dn.is_allowed_in_immutable_directory())
639	dn.raise_error()
640	rep = str(dn)
641	self.failUnless("RW-MUT" in rep)
642	return dn.list()
643	d.addCallback(_created)
644
645	def _check_kids(children):
646	self.failUnlessReallyEqual(set(children.keys()),
647	set([one_nfc, u"two", u"mut", u"mdmf", u"fut", u"fro",
648	u"fut-unic", u"fro-unic", u"empty_litdir", u"tiny_litdir"]))
649	one_node, one_metadata = children[one_nfc]
650	two_node, two_metadata = children[u"two"]
651	mut_node, mut_metadata = children[u"mut"]
652	mdmf_node, mdmf_metadata = children[u"mdmf"]
653	fut_node, fut_metadata = children[u"fut"]
654	fro_node, fro_metadata = children[u"fro"]
655	futna_node, futna_metadata = children[u"fut-unic"]
656	frona_node, frona_metadata = children[u"fro-unic"]
657	emptylit_node, emptylit_metadata = children[u"empty_litdir"]
658	tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
659
660	self.failUnlessReallyEqual(one_node.get_size(), 3)
661	self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
662	self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
663	self.failUnless(isinstance(one_metadata, dict), one_metadata)
664
665	self.failUnlessReallyEqual(two_node.get_size(), 14861)
666	self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
667	self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
668	self.failUnlessEqual(two_metadata["metakey"], "metavalue")
669
670	self.failUnlessReallyEqual(mut_node.get_uri(), mut_write_uri)
671	self.failUnlessReallyEqual(mut_node.get_readonly_uri(), mut_read_uri)
672	self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
673
674	self.failUnlessReallyEqual(mdmf_node.get_uri(), mdmf_write_uri)
675	self.failUnlessReallyEqual(mdmf_node.get_readonly_uri(), mdmf_read_uri)
676	self.failUnless(isinstance(mdmf_metadata, dict), mdmf_metadata)
677
678	self.failUnless(fut_node.is_unknown())
679	self.failUnlessReallyEqual(fut_node.get_uri(), future_write_uri)
680	self.failUnlessReallyEqual(fut_node.get_readonly_uri(), b"ro." + future_read_uri)
681	self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
682
683	self.failUnless(futna_node.is_unknown())
684	self.failUnlessReallyEqual(futna_node.get_uri(), future_nonascii_write_uri)
685	self.failUnlessReallyEqual(futna_node.get_readonly_uri(), b"ro." + future_nonascii_read_uri)
686	self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
687
688	self.failUnless(fro_node.is_unknown())
689	self.failUnlessReallyEqual(fro_node.get_uri(), b"ro." + future_read_uri)
690	self.failUnlessReallyEqual(fut_node.get_readonly_uri(), b"ro." + future_read_uri)
691	self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
692
693	self.failUnless(frona_node.is_unknown())
694	self.failUnlessReallyEqual(frona_node.get_uri(), b"ro." + future_nonascii_read_uri)
695	self.failUnlessReallyEqual(futna_node.get_readonly_uri(), b"ro." + future_nonascii_read_uri)
696	self.failUnless(isinstance(frona_metadata, dict), frona_metadata)
697
698	self.failIf(emptylit_node.is_unknown())
699	self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
700	self.failIf(tinylit_node.is_unknown())
701	self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
702
703	d2 = defer.succeed(None)
704	d2.addCallback(lambda ignored: emptylit_node.list())
705	d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
706	d2.addCallback(lambda ignored: tinylit_node.list())
707	d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
708	set([u"short"])))
709	d2.addCallback(lambda ignored: tinylit_node.list())
710	d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
711	d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, b"The end."))
712	return d2
713	d.addCallback(_check_kids)
714
715	d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
716	d.addCallback(lambda dn: dn.list())
717	d.addCallback(_check_kids)
718
719	bad_future_node = UnknownNode(future_write_uri, None)
720	bad_kids1 = {one_nfd: (bad_future_node, {})}
721	# This should fail because we don't know how to diminish the future_write_uri
722	# cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
723	d.addCallback(lambda ign:
724	self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
725	"cannot attach unknown",
726	nm.create_new_mutable_directory,
727	bad_kids1))
728	bad_kids2 = {one_nfd: (nm.create_from_cap(one_uri), None)}
729	d.addCallback(lambda ign:
730	self.shouldFail(AssertionError, "bad_kids2",
731	"requires metadata to be a dict",
732	nm.create_new_mutable_directory,
733	bad_kids2))
734	return d
735
736	def _do_basic_test(self, mdmf=False):
737	c = self.g.clients[0]
738	d = None
739	if mdmf:
740	d = c.create_dirnode(version=MDMF_VERSION)
741	else:
742	d = c.create_dirnode()
743	def _done(res):
744	self.failUnless(isinstance(res, dirnode.DirectoryNode))
745	self.failUnless(res.is_mutable())
746	self.failIf(res.is_readonly())
747	self.failIf(res.is_unknown())
748	self.failIf(res.is_allowed_in_immutable_directory())
749	res.raise_error()
750	rep = str(res)
751	self.failUnless("RW-MUT" in rep)
752	d.addCallback(_done)
753	return d
754
755	def test_basic(self):
756	self.basedir = "dirnode/Dirnode/test_basic"
757	self.set_up_grid(oneshare=True)
758	return self._do_basic_test()
759
760	def test_basic_mdmf(self):
761	self.basedir = "dirnode/Dirnode/test_basic_mdmf"
762	self.set_up_grid(oneshare=True)
763	return self._do_basic_test(mdmf=True)
764
765	def test_initial_children(self):
766	self.basedir = "dirnode/Dirnode/test_initial_children"
767	self.set_up_grid(oneshare=True)
768	return self._do_initial_children_test()
769
770	def test_immutable(self):
771	self.basedir = "dirnode/Dirnode/test_immutable"
772	self.set_up_grid(oneshare=True)
773	c = self.g.clients[0]
774	nm = c.nodemaker
775
776	kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
777	u"two": (nm.create_from_cap(setup_py_uri),
778	{"metakey": "metavalue"}),
779	u"fut": (nm.create_from_cap(None, future_read_uri), {}),
780	u"futna": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
781	u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
782	u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
783	}
784	d = c.create_immutable_dirnode(kids)
785
786	def _created(dn):
787	self.failUnless(isinstance(dn, dirnode.DirectoryNode))
788	self.failIf(dn.is_mutable())
789	self.failUnless(dn.is_readonly())
790	self.failIf(dn.is_unknown())
791	self.failUnless(dn.is_allowed_in_immutable_directory())
792	dn.raise_error()
793	rep = str(dn)
794	self.failUnless("RO-IMM" in rep)
795	cap = dn.get_cap()
796	self.failUnlessIn(b"CHK", cap.to_string())
797	self.cap = cap
798	return dn.list()
799	d.addCallback(_created)
800
801	def _check_kids(children):
802	self.failUnlessReallyEqual(set(children.keys()),
803	set([one_nfc, u"two", u"fut", u"futna", u"empty_litdir", u"tiny_litdir"]))
804	one_node, one_metadata = children[one_nfc]
805	two_node, two_metadata = children[u"two"]
806	fut_node, fut_metadata = children[u"fut"]
807	futna_node, futna_metadata = children[u"futna"]
808	emptylit_node, emptylit_metadata = children[u"empty_litdir"]
809	tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
810
811	self.failUnlessReallyEqual(one_node.get_size(), 3)
812	self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
813	self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
814	self.failUnless(isinstance(one_metadata, dict), one_metadata)
815
816	self.failUnlessReallyEqual(two_node.get_size(), 14861)
817	self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
818	self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
819	self.failUnlessEqual(two_metadata["metakey"], "metavalue")
820
821	self.failUnless(fut_node.is_unknown())
822	self.failUnlessReallyEqual(fut_node.get_uri(), b"imm." + future_read_uri)
823	self.failUnlessReallyEqual(fut_node.get_readonly_uri(), b"imm." + future_read_uri)
824	self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
825
826	self.failUnless(futna_node.is_unknown())
827	self.failUnlessReallyEqual(futna_node.get_uri(), b"imm." + future_nonascii_read_uri)
828	self.failUnlessReallyEqual(futna_node.get_readonly_uri(), b"imm." + future_nonascii_read_uri)
829	self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
830
831	self.failIf(emptylit_node.is_unknown())
832	self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
833	self.failIf(tinylit_node.is_unknown())
834	self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
835
836	d2 = defer.succeed(None)
837	d2.addCallback(lambda ignored: emptylit_node.list())
838	d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
839	d2.addCallback(lambda ignored: tinylit_node.list())
840	d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
841	set([u"short"])))
842	d2.addCallback(lambda ignored: tinylit_node.list())
843	d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
844	d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, b"The end."))
845	return d2
846
847	d.addCallback(_check_kids)
848
849	d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
850	d.addCallback(lambda dn: dn.list())
851	d.addCallback(_check_kids)
852
853	bad_future_node1 = UnknownNode(future_write_uri, None)
854	bad_kids1 = {one_nfd: (bad_future_node1, {})}
855	d.addCallback(lambda ign:
856	self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
857	"cannot attach unknown",
858	c.create_immutable_dirnode,
859	bad_kids1))
860	bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
861	bad_kids2 = {one_nfd: (bad_future_node2, {})}
862	d.addCallback(lambda ign:
863	self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
864	"is not allowed in an immutable directory",
865	c.create_immutable_dirnode,
866	bad_kids2))
867	bad_kids3 = {one_nfd: (nm.create_from_cap(one_uri), None)}
868	d.addCallback(lambda ign:
869	self.shouldFail(AssertionError, "bad_kids3",
870	"requires metadata to be a dict",
871	c.create_immutable_dirnode,
872	bad_kids3))
873	bad_kids4 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
874	d.addCallback(lambda ign:
875	self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
876	"is not allowed in an immutable directory",
877	c.create_immutable_dirnode,
878	bad_kids4))
879	bad_kids5 = {one_nfd: (nm.create_from_cap(mut_read_uri), {})}
880	d.addCallback(lambda ign:
881	self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
882	"is not allowed in an immutable directory",
883	c.create_immutable_dirnode,
884	bad_kids5))
885	bad_kids6 = {one_nfd: (nm.create_from_cap(mdmf_write_uri), {})}
886	d.addCallback(lambda ign:
887	self.shouldFail(MustBeDeepImmutableError, "bad_kids6",
888	"is not allowed in an immutable directory",
889	c.create_immutable_dirnode,
890	bad_kids6))
891	bad_kids7 = {one_nfd: (nm.create_from_cap(mdmf_read_uri), {})}
892	d.addCallback(lambda ign:
893	self.shouldFail(MustBeDeepImmutableError, "bad_kids7",
894	"is not allowed in an immutable directory",
895	c.create_immutable_dirnode,
896	bad_kids7))
897	d.addCallback(lambda ign: c.create_immutable_dirnode({}))
898	def _created_empty(dn):
899	self.failUnless(isinstance(dn, dirnode.DirectoryNode))
900	self.failIf(dn.is_mutable())
901	self.failUnless(dn.is_readonly())
902	self.failIf(dn.is_unknown())
903	self.failUnless(dn.is_allowed_in_immutable_directory())
904	dn.raise_error()
905	rep = str(dn)
906	self.failUnless("RO-IMM" in rep)
907	cap = dn.get_cap()
908	self.failUnlessIn(b"LIT", cap.to_string())
909	self.failUnlessReallyEqual(cap.to_string(), b"URI:DIR2-LIT:")
910	self.cap = cap
911	return dn.list()
912	d.addCallback(_created_empty)
913	d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
914	smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
915	d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
916	def _created_small(dn):
917	self.failUnless(isinstance(dn, dirnode.DirectoryNode))
918	self.failIf(dn.is_mutable())
919	self.failUnless(dn.is_readonly())
920	self.failIf(dn.is_unknown())
921	self.failUnless(dn.is_allowed_in_immutable_directory())
922	dn.raise_error()
923	rep = str(dn)
924	self.failUnless("RO-IMM" in rep)
925	cap = dn.get_cap()
926	self.failUnlessIn(b"LIT", cap.to_string())
927	self.failUnlessReallyEqual(cap.to_string(),
928	b"URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
929	self.cap = cap
930	return dn.list()
931	d.addCallback(_created_small)
932	d.addCallback(lambda kids: self.failUnlessReallyEqual(list(kids.keys()), [u"o"]))
933
934	# now test n.create_subdirectory(mutable=False)
935	d.addCallback(lambda ign: c.create_dirnode())
936	def _made_parent(n):
937	d = n.create_subdirectory(u"subdir", kids, mutable=False)
938	d.addCallback(lambda sd: sd.list())
939	d.addCallback(_check_kids)
940	d.addCallback(lambda ign: n.list())
941	d.addCallback(lambda children:
942	self.failUnlessReallyEqual(list(children.keys()), [u"subdir"]))
943	d.addCallback(lambda ign: n.get(u"subdir"))
944	d.addCallback(lambda sd: sd.list())
945	d.addCallback(_check_kids)
946	d.addCallback(lambda ign: n.get(u"subdir"))
947	d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
948	bad_kids8 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
949	d.addCallback(lambda ign:
950	self.shouldFail(MustBeDeepImmutableError, "bad_kids8",
951	"is not allowed in an immutable directory",
952	n.create_subdirectory,
953	u"sub2", bad_kids8, mutable=False))
954	bad_kids9 = {one_nfd: (nm.create_from_cap(mdmf_write_uri), {})}
955	d.addCallback(lambda ign:
956	self.shouldFail(MustBeDeepImmutableError, "bad_kids9",
957	"is not allowed in an immutable directory",
958	n.create_subdirectory,
959	u"sub2", bad_kids9, mutable=False))
960	return d
961	d.addCallback(_made_parent)
962	return d
963
964	def test_directory_representation(self):
965	self.basedir = "dirnode/Dirnode/test_directory_representation"
966	self.set_up_grid(oneshare=True)
967	c = self.g.clients[0]
968	nm = c.nodemaker
969
970	# This test checks that any trailing spaces in URIs are retained in the
971	# encoded directory, but stripped when we get them out of the directory.
972	# See ticket #925 for why we want that.
973	# It also tests that we store child names as UTF-8 NFC, and normalize
974	# them again when retrieving them.
975
976	stripped_write_uri = b"lafs://from_the_future\t"
977	stripped_read_uri = b"lafs://readonly_from_the_future\t"
978	spacedout_write_uri = stripped_write_uri + b" "
979	spacedout_read_uri = stripped_read_uri + b" "
980
981	child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
982	self.failUnlessReallyEqual(child.get_write_uri(), spacedout_write_uri)
983	self.failUnlessReallyEqual(child.get_readonly_uri(), b"ro." + spacedout_read_uri)
984
985	child_dottedi = u"ch\u0131\u0307ld"
986
987	kids_in = {child_dottedi: (child, {}), one_nfd: (child, {})}
988	kids_out = {child_dottedi: (child, {}), one_nfc: (child, {})}
989	kids_norm = {u"child": (child, {}), one_nfc: (child, {})}
990	d = c.create_dirnode(kids_in)
991
992	def _created(dn):
993	self.failUnless(isinstance(dn, dirnode.DirectoryNode))
994	self.failUnless(dn.is_mutable())
995	self.failIf(dn.is_readonly())
996	dn.raise_error()
997	self.cap = dn.get_cap()
998	self.rootnode = dn
999	return dn._node.download_best_version()
1000	d.addCallback(_created)
1001
1002	def _check_data(data):
1003	# Decode the netstring representation of the directory to check that the
1004	# spaces are retained when the URIs are stored, and that the names are stored
1005	# as NFC.
1006	position = 0
1007	numkids = 0
1008	while position < len(data):
1009	entries, position = split_netstring(data, 1, position)
1010	entry = entries[0]
1011	(name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
1012	name = name_utf8.decode("utf-8")
1013	rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
1014	self.failUnlessIn(name, kids_out)
1015	(expected_child, ign) = kids_out[name]
1016	self.failUnlessReallyEqual(rw_uri, expected_child.get_write_uri())
1017	self.failUnlessReallyEqual(b"ro." + ro_uri, expected_child.get_readonly_uri())
1018	numkids += 1
1019
1020	self.failUnlessReallyEqual(numkids, len(kids_out))
1021	return self.rootnode
1022	d.addCallback(_check_data)
1023
1024	# Mock up a hypothetical future version of Unicode that adds a canonical equivalence
1025	# between dotless-i + dot-above, and 'i'. That would actually be prohibited by the
1026	# stability rules, but similar additions involving currently-unassigned characters
1027	# would not be.
1028	old_normalize = unicodedata.normalize
1029	def future_normalize(form, s):
1030	assert form == 'NFC', form
1031	return old_normalize(form, s).replace(u"\u0131\u0307", u"i")
1032
1033	def _list(node):
1034	unicodedata.normalize = future_normalize
1035	d2 = node.list()
1036	def _undo_mock(res):
1037	unicodedata.normalize = old_normalize
1038	return res
1039	d2.addBoth(_undo_mock)
1040	return d2
1041	d.addCallback(_list)
1042
1043	def _check_kids(children):
1044	# Now when we use the real directory listing code, the trailing spaces
1045	# should have been stripped (and "ro." should have been prepended to the
1046	# ro_uri, since it's unknown). Also the dotless-i + dot-above should have been
1047	# normalized to 'i'.
1048
1049	self.failUnlessReallyEqual(set(children.keys()), set(kids_norm.keys()))
1050	child_node, child_metadata = children[u"child"]
1051
1052	self.failUnlessReallyEqual(child_node.get_write_uri(), stripped_write_uri)
1053	self.failUnlessReallyEqual(child_node.get_readonly_uri(), b"ro." + stripped_read_uri)
1054	d.addCallback(_check_kids)
1055
1056	d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
1057	d.addCallback(_list)
1058	d.addCallback(_check_kids) # again with dirnode recreated from cap
1059	return d
1060
1061	def test_check(self):
1062	self.basedir = "dirnode/Dirnode/test_check"
1063	self.set_up_grid(oneshare=True)
1064	c = self.g.clients[0]
1065	d = c.create_dirnode()
1066	d.addCallback(lambda dn: dn.check(Monitor()))
1067	def _done(res):
1068	self.failUnless(res.is_healthy())
1069	d.addCallback(_done)
1070	return d
1071
1072	def _test_deepcheck_create(self, version=SDMF_VERSION):
1073	# create a small tree with a loop, and some non-directories
1074	# root/
1075	# root/subdir/
1076	# root/subdir/file1
1077	# root/subdir/link -> root
1078	# root/rodir
1079	c = self.g.clients[0]
1080	d = c.create_dirnode(version=version)
1081	def _created_root(rootnode):
1082	self._rootnode = rootnode
1083	self.failUnlessEqual(rootnode._node.get_version(), version)
1084	return rootnode.create_subdirectory(u"subdir")
1085	d.addCallback(_created_root)
1086	def _created_subdir(subdir):
1087	self._subdir = subdir
1088	d = subdir.add_file(u"file1", upload.Data(b"data"*100, None))
1089	d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
1090	d.addCallback(lambda res: c.create_dirnode())
1091	d.addCallback(lambda dn:
1092	self._rootnode.set_uri(u"rodir",
1093	dn.get_uri(),
1094	dn.get_readonly_uri()))
1095	return d
1096	d.addCallback(_created_subdir)
1097	def _done(res):
1098	return self._rootnode
1099	d.addCallback(_done)
1100	return d
1101
1102	def test_deepcheck(self):
1103	self.basedir = "dirnode/Dirnode/test_deepcheck"
1104	self.set_up_grid(oneshare=True)
1105	d = self._test_deepcheck_create()
1106	d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1107	def _check_results(r):
1108	self.failUnless(IDeepCheckResults.providedBy(r))
1109	c = r.get_counters()
1110	self.failUnlessReallyEqual(c,
1111	{"count-objects-checked": 4,
1112	"count-objects-healthy": 4,
1113	"count-objects-unhealthy": 0,
1114	"count-objects-unrecoverable": 0,
1115	"count-corrupt-shares": 0,
1116	})
1117	self.failIf(r.get_corrupt_shares())
1118	self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1119	d.addCallback(_check_results)
1120	return d
1121
1122	def test_deepcheck_cachemisses(self):
1123	self.basedir = "dirnode/Dirnode/test_mdmf_cachemisses"
1124	self.set_up_grid(oneshare=True)
1125	d = self._test_deepcheck_create()
1126	# Clear the counters and set the rootnode
1127	d.addCallback(lambda rootnode:
1128	not [ss._clear_counters() for ss
1129	in self.g.wrappers_by_id.values()] or rootnode)
1130	d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1131	def _check(ign):
1132	count = sum([ss.counter_by_methname['slot_readv']
1133	for ss in self.g.wrappers_by_id.values()])
1134	self.failIf(count > 60, 'Expected only 60 cache misses,'
1135	'unfortunately there were %d' % (count,))
1136	d.addCallback(_check)
1137	return d
1138
1139	def test_deepcheck_mdmf(self):
1140	self.basedir = "dirnode/Dirnode/test_deepcheck_mdmf"
1141	self.set_up_grid(oneshare=True)
1142	d = self._test_deepcheck_create(MDMF_VERSION)
1143	d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1144	def _check_results(r):
1145	self.failUnless(IDeepCheckResults.providedBy(r))
1146	c = r.get_counters()
1147	self.failUnlessReallyEqual(c,
1148	{"count-objects-checked": 4,
1149	"count-objects-healthy": 4,
1150	"count-objects-unhealthy": 0,
1151	"count-objects-unrecoverable": 0,
1152	"count-corrupt-shares": 0,
1153	})
1154	self.failIf(r.get_corrupt_shares())
1155	self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1156	d.addCallback(_check_results)
1157	return d
1158
1159	def test_deepcheck_and_repair(self):
1160	self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
1161	self.set_up_grid(oneshare=True)
1162	d = self._test_deepcheck_create()
1163	d.addCallback(lambda rootnode:
1164	rootnode.start_deep_check_and_repair().when_done())
1165	def _check_results(r):
1166	self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
1167	c = r.get_counters()
1168	self.failUnlessReallyEqual(c,
1169	{"count-objects-checked": 4,
1170	"count-objects-healthy-pre-repair": 4,
1171	"count-objects-unhealthy-pre-repair": 0,
1172	"count-objects-unrecoverable-pre-repair": 0,
1173	"count-corrupt-shares-pre-repair": 0,
1174	"count-objects-healthy-post-repair": 4,
1175	"count-objects-unhealthy-post-repair": 0,
1176	"count-objects-unrecoverable-post-repair": 0,
1177	"count-corrupt-shares-post-repair": 0,
1178	"count-repairs-attempted": 0,
1179	"count-repairs-successful": 0,
1180	"count-repairs-unsuccessful": 0,
1181	})
1182	self.failIf(r.get_corrupt_shares())
1183	self.failIf(r.get_remaining_corrupt_shares())
1184	self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1185	d.addCallback(_check_results)
1186	return d
1187
1188	def test_deepcheck_and_repair_mdmf(self):
1189	self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair_mdmf"
1190	self.set_up_grid(oneshare=True)
1191	d = self._test_deepcheck_create(version=MDMF_VERSION)
1192	d.addCallback(lambda rootnode:
1193	rootnode.start_deep_check_and_repair().when_done())
1194	def _check_results(r):
1195	self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
1196	c = r.get_counters()
1197	self.failUnlessReallyEqual(c,
1198	{"count-objects-checked": 4,
1199	"count-objects-healthy-pre-repair": 4,
1200	"count-objects-unhealthy-pre-repair": 0,
1201	"count-objects-unrecoverable-pre-repair": 0,
1202	"count-corrupt-shares-pre-repair": 0,
1203	"count-objects-healthy-post-repair": 4,
1204	"count-objects-unhealthy-post-repair": 0,
1205	"count-objects-unrecoverable-post-repair": 0,
1206	"count-corrupt-shares-post-repair": 0,
1207	"count-repairs-attempted": 0,
1208	"count-repairs-successful": 0,
1209	"count-repairs-unsuccessful": 0,
1210	})
1211	self.failIf(r.get_corrupt_shares())
1212	self.failIf(r.get_remaining_corrupt_shares())
1213	self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1214	d.addCallback(_check_results)
1215	return d
1216
1217	def _mark_file_bad(self, rootnode):
1218	self.delete_shares_numbered(rootnode.get_uri(), [0])
1219	return rootnode
1220
1221	def test_deepcheck_problems(self):
1222	self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
1223	self.set_up_grid()
1224	d = self._test_deepcheck_create()
1225	d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
1226	d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1227	def _check_results(r):
1228	c = r.get_counters()
1229	self.failUnlessReallyEqual(c,
1230	{"count-objects-checked": 4,
1231	"count-objects-healthy": 3,
1232	"count-objects-unhealthy": 1,
1233	"count-objects-unrecoverable": 0,
1234	"count-corrupt-shares": 0,
1235	})
1236	#self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
1237	d.addCallback(_check_results)
1238	return d
1239
1240	def test_deepcheck_problems_mdmf(self):
1241	self.basedir = "dirnode/Dirnode/test_deepcheck_problems_mdmf"
1242	self.set_up_grid()
1243	d = self._test_deepcheck_create(version=MDMF_VERSION)
1244	d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
1245	d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1246	def _check_results(r):
1247	c = r.get_counters()
1248	self.failUnlessReallyEqual(c,
1249	{"count-objects-checked": 4,
1250	"count-objects-healthy": 3,
1251	"count-objects-unhealthy": 1,
1252	"count-objects-unrecoverable": 0,
1253	"count-corrupt-shares": 0,
1254	})
1255	#self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
1256	d.addCallback(_check_results)
1257	return d
1258
1259	def _do_readonly_test(self, version=SDMF_VERSION):
1260	c = self.g.clients[0]
1261	nm = c.nodemaker
1262	filecap = make_chk_file_uri(1234)
1263	filenode = nm.create_from_cap(filecap)
1264	uploadable = upload.Data(b"some data", convergence=b"some convergence string")
1265
1266	d = c.create_dirnode(version=version)
1267	def _created(rw_dn):
1268	backing_node = rw_dn._node
1269	self.failUnlessEqual(backing_node.get_version(), version)
1270	d2 = rw_dn.set_uri(u"child", filecap, filecap)
1271	d2.addCallback(lambda res: rw_dn)
1272	return d2
1273	d.addCallback(_created)
1274
1275	def _ready(rw_dn):
1276	ro_uri = rw_dn.get_readonly_uri()
1277	ro_dn = c.create_node_from_uri(ro_uri)
1278	self.failUnless(ro_dn.is_readonly())
1279	self.failUnless(ro_dn.is_mutable())
1280	self.failIf(ro_dn.is_unknown())
1281	self.failIf(ro_dn.is_allowed_in_immutable_directory())
1282	ro_dn.raise_error()
1283
1284	self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1285	ro_dn.set_uri, u"newchild", filecap, filecap)
1286	self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1287	ro_dn.set_node, u"newchild", filenode)
1288	self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
1289	ro_dn.set_nodes, { u"newchild": (filenode, None) })
1290	self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1291	ro_dn.add_file, u"newchild", uploadable)
1292	self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1293	ro_dn.delete, u"child")
1294	self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1295	ro_dn.create_subdirectory, u"newchild")
1296	self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
1297	ro_dn.set_metadata_for, u"child", {})
1298	self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1299	ro_dn.move_child_to, u"child", rw_dn)
1300	self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1301	rw_dn.move_child_to, u"child", ro_dn)
1302	return ro_dn.list()
1303	d.addCallback(_ready)
1304	def _listed(children):
1305	self.failUnless(u"child" in children)
1306	d.addCallback(_listed)
1307	return d
1308
1309	def test_readonly(self):
1310	self.basedir = "dirnode/Dirnode/test_readonly"
1311	self.set_up_grid(oneshare=True)
1312	return self._do_readonly_test()
1313
1314	def test_readonly_mdmf(self):
1315	self.basedir = "dirnode/Dirnode/test_readonly_mdmf"
1316	self.set_up_grid(oneshare=True)
1317	return self._do_readonly_test(version=MDMF_VERSION)
1318
1319	def failUnlessGreaterThan(self, a, b):
1320	self.failUnless(a > b, "%r should be > %r" % (a, b))
1321
1322	def failUnlessGreaterOrEqualThan(self, a, b):
1323	self.failUnless(a >= b, "%r should be >= %r" % (a, b))
1324
1325	def test_create(self):
1326	self.basedir = "dirnode/Dirnode/test_create"
1327	self.set_up_grid(oneshare=True)
1328	return self._do_create_test()
1329
1330	def test_update_metadata(self):
1331	(t1, t2, t3) = (626644800.0, 634745640.0, 892226160.0)
1332
1333	md1 = dirnode.update_metadata({"ctime": t1}, {}, t2)
1334	self.failUnlessEqual(md1, {"tahoe":{"linkcrtime": t1, "linkmotime": t2}})
1335
1336	md2 = dirnode.update_metadata(md1, {"key": "value", "tahoe": {"bad": "mojo"}}, t3)
1337	self.failUnlessEqual(md2, {"key": "value",
1338	"tahoe":{"linkcrtime": t1, "linkmotime": t3}})
1339
1340	md3 = dirnode.update_metadata({}, None, t3)
1341	self.failUnlessEqual(md3, {"tahoe":{"linkcrtime": t3, "linkmotime": t3}})
1342
1343	md4 = dirnode.update_metadata({}, {"bool": True, "number": 42}, t1)
1344	self.failUnlessEqual(md4, {"bool": True, "number": 42,
1345	"tahoe":{"linkcrtime": t1, "linkmotime": t1}})
1346
1347	def _do_create_subdirectory_test(self, version=SDMF_VERSION):
1348	c = self.g.clients[0]
1349	nm = c.nodemaker
1350
1351	d = c.create_dirnode(version=version)
1352	def _then(n):
1353	# /
1354	self.rootnode = n
1355	fake_file_uri = make_mutable_file_uri()
1356	other_file_uri = make_mutable_file_uri()
1357	md = {"metakey": "metavalue"}
1358	kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1359	u"kid2": (nm.create_from_cap(other_file_uri), md),
1360	}
1361	d = n.create_subdirectory(u"subdir", kids,
1362	mutable_version=version)
1363	def _check(sub):
1364	d = n.get_child_at_path(u"subdir")
1365	d.addCallback(lambda sub2: self.failUnlessReallyEqual(sub2.get_uri(),
1366	sub.get_uri()))
1367	d.addCallback(lambda ign: sub.list())
1368	return d
1369	d.addCallback(_check)
1370	def _check_kids(kids2):
1371	self.failUnlessEqual(set(kids.keys()), set(kids2.keys()))
1372	self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1373	d.addCallback(_check_kids)
1374	return d
1375	d.addCallback(_then)
1376	return d
1377
1378	def test_create_subdirectory(self):
1379	self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1380	self.set_up_grid(oneshare=True)
1381	return self._do_create_subdirectory_test()
1382
1383	def test_create_subdirectory_mdmf(self):
1384	self.basedir = "dirnode/Dirnode/test_create_subdirectory_mdmf"
1385	self.set_up_grid(oneshare=True)
1386	return self._do_create_subdirectory_test(version=MDMF_VERSION)
1387
1388	def test_create_mdmf(self):
1389	self.basedir = "dirnode/Dirnode/test_mdmf"
1390	self.set_up_grid(oneshare=True)
1391	return self._do_create_test(mdmf=True)
1392
1393	def test_mdmf_initial_children(self):
1394	self.basedir = "dirnode/Dirnode/test_mdmf"
1395	self.set_up_grid(oneshare=True)
1396	return self._do_initial_children_test(mdmf=True)
1397
1398	class MinimalFakeMutableFile:
1399	def get_writekey(self):
1400	return b"writekey"
1401
1402	class Packing(testutil.ReallyEqualMixin, unittest.TestCase):
1403	# This is a base32-encoded representation of the directory tree
1404	# root/file1
1405	# root/file2
1406	# root/file3
1407	# as represented after being fed to _pack_contents.
1408	# We have it here so we can decode it, feed it to
1409	# _unpack_contents, and verify that _unpack_contents
1410	# works correctly.
1411
1412	known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1413
1414	def test_unpack_and_pack_behavior(self):
1415	known_tree = b32decode(self.known_tree)
1416	nodemaker = NodeMaker(None, None, None,
1417	None, None,
1418	{"k": 3, "n": 10}, None, None)
1419	write_uri = b"URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1420	filenode = nodemaker.create_from_cap(write_uri)
1421	node = dirnode.DirectoryNode(filenode, nodemaker, None)
1422	children = node._unpack_contents(known_tree)
1423	self._check_children(children)
1424
1425	packed_children = node._pack_contents(children)
1426	children = node._unpack_contents(packed_children)
1427	self._check_children(children)
1428
1429	def _check_children(self, children):
1430	# Are all the expected child nodes there?
1431	self.failUnless(u'file1' in children)
1432	self.failUnless(u'file2' in children)
1433	self.failUnless(u'file3' in children)
1434
1435	# Are the metadata for child 3 right?
1436	file3_rocap = b"URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1437	file3_rwcap = b"URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1438	file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1439	self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1440	self.failUnlessReallyEqual(file3_rocap,
1441	children[u'file3'][0].get_readonly_uri())
1442	self.failUnlessReallyEqual(file3_rwcap,
1443	children[u'file3'][0].get_uri())
1444
1445	# Are the metadata for child 2 right?
1446	file2_rocap = b"URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1447	file2_rwcap = b"URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1448	file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1449	self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1450	self.failUnlessReallyEqual(file2_rocap,
1451	children[u'file2'][0].get_readonly_uri())
1452	self.failUnlessReallyEqual(file2_rwcap,
1453	children[u'file2'][0].get_uri())
1454
1455	# Are the metadata for child 1 right?
1456	file1_rocap = b"URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1457	file1_rwcap = b"URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1458	file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1459	self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1460	self.failUnlessReallyEqual(file1_rocap,
1461	children[u'file1'][0].get_readonly_uri())
1462	self.failUnlessReallyEqual(file1_rwcap,
1463	children[u'file1'][0].get_uri())
1464
1465	def _make_kids(self, nm, which):
1466	caps = {"imm": b"URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1467	"lit": b"URI:LIT:n5xgk", # LIT for "one"
1468	"write": b"URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1469	"read": b"URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1470	"dirwrite": b"URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1471	"dirread": b"URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1472	}
1473	kids = {}
1474	for name in which:
1475	kids[str(name)] = (nm.create_from_cap(caps[name]), {})
1476	return kids
1477
1478	def test_pack_unpack_unknown(self):
1479	"""
1480	Minimal testing for roundtripping unknown URIs.
1481	"""
1482	nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None)
1483	fn = MinimalFakeMutableFile()
1484	# UnknownNode has massively complex rules about when it's an error.
1485	# Just force it not to be an error.
1486	unknown_rw = UnknownNode(b"whatevs://write", None)
1487	unknown_rw.error = None
1488	unknown_ro = UnknownNode(None, b"whatevs://readonly")
1489	unknown_ro.error = None
1490	kids = {
1491	"unknown_rw": (unknown_rw, {}),
1492	"unknown_ro": (unknown_ro, {})
1493	}
1494	packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
1495
1496	write_uri = b"URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1497	filenode = nm.create_from_cap(write_uri)
1498	dn = dirnode.DirectoryNode(filenode, nm, None)
1499	unkids = dn._unpack_contents(packed)
1500	self.assertEqual(kids, unkids)
1501
1502	@given(text(min_size=1, max_size=20))
1503	def test_pack_unpack_unicode_hypothesis(self, name):
1504	"""
1505	pack -> unpack results in the same objects (with a unicode name)
1506	"""
1507	nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None)
1508	fn = MinimalFakeMutableFile()
1509
1510	# FIXME TODO: we shouldn't have to do this out here, but
1511	# Hypothesis found that a name with "\x2000" does not make the
1512	# round-trip properly .. so for now we'll only give the packer
1513	# normalized names.
1514	# See also:
1515	# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2606
1516	# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1076
1517	name = unicodedata.normalize('NFC', name)
1518
1519	kids = {
1520	name: (LiteralFileNode(uri.from_string(one_uri)), {}),
1521	}
1522	packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
1523	write_uri = b"URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1524	filenode = nm.create_from_cap(write_uri)
1525	dn = dirnode.DirectoryNode(filenode, nm, None)
1526	unkids = dn._unpack_contents(packed)
1527	self.assertEqual(kids, unkids)
1528
1529	def test_deep_immutable(self):
1530	nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None)
1531	fn = MinimalFakeMutableFile()
1532
1533	kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1534	"dirwrite", "dirread"])
1535	packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
1536	self.failUnlessIn(b"lit", packed)
1537
1538	kids = self._make_kids(nm, ["imm", "lit"])
1539	packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=True)
1540	self.failUnlessIn(b"lit", packed)
1541
1542	kids = self._make_kids(nm, ["imm", "lit", "write"])
1543	self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1544	dirnode.pack_children,
1545	kids, fn.get_writekey(), deep_immutable=True)
1546
1547	# read-only is not enough: all children must be immutable
1548	kids = self._make_kids(nm, ["imm", "lit", "read"])
1549	self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1550	dirnode.pack_children,
1551	kids, fn.get_writekey(), deep_immutable=True)
1552
1553	kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1554	self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1555	dirnode.pack_children,
1556	kids, fn.get_writekey(), deep_immutable=True)
1557
1558	kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1559	self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1560	dirnode.pack_children,
1561	kids, fn.get_writekey(), deep_immutable=True)
1562
1563	@implementer(IMutableFileNode)
1564	class FakeMutableFile(object): # type: ignore # incomplete implementation
1565	counter = 0
1566	def __init__(self, initial_contents=b""):
1567	data = self._get_initial_contents(initial_contents)
1568	self.data = data.read(data.get_size())
1569	self.data = b"".join(self.data)
1570
1571	counter = FakeMutableFile.counter
1572	FakeMutableFile.counter += 1
1573	writekey = hashutil.ssk_writekey_hash(b"%d" % counter)
1574	fingerprint = hashutil.ssk_pubkey_fingerprint_hash(b"%d" % counter)
1575	self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1576
1577	def _get_initial_contents(self, contents):
1578	if isinstance(contents, bytes):
1579	return contents
1580	if contents is None:
1581	return b""
1582	assert callable(contents), "%s should be callable, not %s" % \
1583	(contents, type(contents))
1584	return contents(self)
1585
1586	def get_cap(self):
1587	return self.uri
1588
1589	def get_uri(self):
1590	return self.uri.to_string()
1591
1592	def get_write_uri(self):
1593	return self.uri.to_string()
1594
1595	def download_best_version(self):
1596	return defer.succeed(self.data)
1597
1598	def get_writekey(self):
1599	return b"writekey"
1600
1601	def is_readonly(self):
1602	return False
1603
1604	def is_mutable(self):
1605	return True
1606
1607	def is_unknown(self):
1608	return False
1609
1610	def is_allowed_in_immutable_directory(self):
1611	return False
1612
1613	def raise_error(self):
1614	pass
1615
1616	def modify(self, modifier):
1617	data = modifier(self.data, None, True)
1618	self.data = data
1619	return defer.succeed(None)
1620
1621	class FakeNodeMaker(NodeMaker):
1622	def create_mutable_file(self, contents=b"", keysize=None, version=None, keypair=None):
1623	assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs"
1624	return defer.succeed(FakeMutableFile(contents))
1625
1626	class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573
1627	def __init__(self):
1628	self.nodemaker = FakeNodeMaker(None, None, None,
1629	None, None,
1630	{"k":3,"n":10}, None, None)
1631	def create_node_from_uri(self, rwcap, rocap):
1632	return self.nodemaker.create_from_cap(rwcap, rocap)
1633
1634	class Dirnode2(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase):
1635	def setUp(self):
1636	client = FakeClient2()
1637	self.nodemaker = client.nodemaker
1638
1639	def test_from_future(self):
1640	# Create a mutable directory that contains unknown URI types, and make sure
1641	# we tolerate them properly.
1642	d = self.nodemaker.create_new_mutable_directory()
1643	future_write_uri = u"x-tahoe-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
1644	future_read_uri = u"x-tahoe-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
1645	future_imm_uri = u"x-tahoe-crazy-immutable://I_am_from_the_future_imm_\u263A".encode('utf-8')
1646	future_node = UnknownNode(future_write_uri, future_read_uri)
1647	def _then(n):
1648	self._node = n
1649	return n.set_node(u"future", future_node)
1650	d.addCallback(_then)
1651
1652	# We should be prohibited from adding an unknown URI to a directory
1653	# just in the rw_uri slot, since we don't know how to diminish the cap
1654	# to a readcap (for the ro_uri slot).
1655	d.addCallback(lambda ign:
1656	self.shouldFail(MustNotBeUnknownRWError,
1657	"copy unknown",
1658	"cannot attach unknown rw cap as child",
1659	self._node.set_uri, u"add",
1660	future_write_uri, None))
1661
1662	# However, we should be able to add both rw_uri and ro_uri as a pair of
1663	# unknown URIs.
1664	d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1665	future_write_uri, future_read_uri))
1666
1667	# and to add an URI prefixed with "ro." or "imm." when it is given in a
1668	# write slot (or URL parameter).
1669	d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1670	b"ro." + future_read_uri, None))
1671	d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1672	b"imm." + future_imm_uri, None))
1673
1674	d.addCallback(lambda ign: self._node.list())
1675	def _check(children):
1676	self.failUnlessReallyEqual(len(children), 4)
1677	(fn, metadata) = children[u"future"]
1678	self.failUnless(isinstance(fn, UnknownNode), fn)
1679	self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1680	self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1681	self.failUnlessReallyEqual(fn.get_readonly_uri(), b"ro." + future_read_uri)
1682
1683	(fn2, metadata2) = children[u"add-pair"]
1684	self.failUnless(isinstance(fn2, UnknownNode), fn2)
1685	self.failUnlessReallyEqual(fn2.get_uri(), future_write_uri)
1686	self.failUnlessReallyEqual(fn2.get_write_uri(), future_write_uri)
1687	self.failUnlessReallyEqual(fn2.get_readonly_uri(), b"ro." + future_read_uri)
1688
1689	(fn3, metadata3) = children[u"add-ro"]
1690	self.failUnless(isinstance(fn3, UnknownNode), fn3)
1691	self.failUnlessReallyEqual(fn3.get_uri(), b"ro." + future_read_uri)
1692	self.failUnlessReallyEqual(fn3.get_write_uri(), None)
1693	self.failUnlessReallyEqual(fn3.get_readonly_uri(), b"ro." + future_read_uri)
1694
1695	(fn4, metadata4) = children[u"add-imm"]
1696	self.failUnless(isinstance(fn4, UnknownNode), fn4)
1697	self.failUnlessReallyEqual(fn4.get_uri(), b"imm." + future_imm_uri)
1698	self.failUnlessReallyEqual(fn4.get_write_uri(), None)
1699	self.failUnlessReallyEqual(fn4.get_readonly_uri(), b"imm." + future_imm_uri)
1700
1701	# We should also be allowed to copy the "future" UnknownNode, because
1702	# it contains all the information that was in the original directory
1703	# (readcap and writecap), so we're preserving everything.
1704	return self._node.set_node(u"copy", fn)
1705	d.addCallback(_check)
1706
1707	d.addCallback(lambda ign: self._node.list())
1708	def _check2(children):
1709	self.failUnlessReallyEqual(len(children), 5)
1710	(fn, metadata) = children[u"copy"]
1711	self.failUnless(isinstance(fn, UnknownNode), fn)
1712	self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1713	self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1714	self.failUnlessReallyEqual(fn.get_readonly_uri(), b"ro." + future_read_uri)
1715	d.addCallback(_check2)
1716	return d
1717
1718	def test_unknown_strip_prefix_for_ro(self):
1719	self.failUnlessReallyEqual(strip_prefix_for_ro(b"foo", False), b"foo")
1720	self.failUnlessReallyEqual(strip_prefix_for_ro(b"ro.foo", False), b"foo")
1721	self.failUnlessReallyEqual(strip_prefix_for_ro(b"imm.foo", False), b"imm.foo")
1722	self.failUnlessReallyEqual(strip_prefix_for_ro(b"foo", True), b"foo")
1723	self.failUnlessReallyEqual(strip_prefix_for_ro(b"ro.foo", True), b"foo")
1724	self.failUnlessReallyEqual(strip_prefix_for_ro(b"imm.foo", True), b"foo")
1725
1726	def test_unknownnode(self):
1727	lit_uri = one_uri
1728
1729	# This does not attempt to be exhaustive.
1730	no_no = [# Opaque node, but not an error.
1731	( 0, UnknownNode(None, None)),
1732	( 1, UnknownNode(None, None, deep_immutable=True)),
1733	]
1734	unknown_rw = [# These are errors because we're only given a rw_uri, and we can't
1735	# diminish it.
1736	( 2, UnknownNode(b"foo", None)),
1737	( 3, UnknownNode(b"foo", None, deep_immutable=True)),
1738	( 4, UnknownNode(b"ro.foo", None, deep_immutable=True)),
1739	( 5, UnknownNode(b"ro." + mut_read_uri, None, deep_immutable=True)),
1740	( 5.1, UnknownNode(b"ro." + mdmf_read_uri, None, deep_immutable=True)),
1741	( 6, UnknownNode(b"URI:SSK-RO:foo", None, deep_immutable=True)),
1742	( 7, UnknownNode(b"URI:SSK:foo", None)),
1743	]
1744	must_be_ro = [# These are errors because a readonly constraint is not met.
1745	( 8, UnknownNode(b"ro." + mut_write_uri, None)),
1746	( 8.1, UnknownNode(b"ro." + mdmf_write_uri, None)),
1747	( 9, UnknownNode(None, b"ro." + mut_write_uri)),
1748	( 9.1, UnknownNode(None, b"ro." + mdmf_write_uri)),
1749	]
1750	must_be_imm = [# These are errors because an immutable constraint is not met.
1751	(10, UnknownNode(None, b"ro.URI:SSK-RO:foo", deep_immutable=True)),
1752	(11, UnknownNode(None, b"imm.URI:SSK:foo")),
1753	(12, UnknownNode(None, b"imm.URI:SSK-RO:foo")),
1754	(13, UnknownNode(b"bar", b"ro.foo", deep_immutable=True)),
1755	(14, UnknownNode(b"bar", b"imm.foo", deep_immutable=True)),
1756	(15, UnknownNode(b"bar", b"imm." + lit_uri, deep_immutable=True)),
1757	(16, UnknownNode(b"imm." + mut_write_uri, None)),
1758	(16.1, UnknownNode(b"imm." + mdmf_write_uri, None)),
1759	(17, UnknownNode(b"imm." + mut_read_uri, None)),
1760	(17.1, UnknownNode(b"imm." + mdmf_read_uri, None)),
1761	(18, UnknownNode(b"bar", b"imm.foo")),
1762	]
1763	bad_uri = [# These are errors because the URI is bad once we've stripped the prefix.
1764	(19, UnknownNode(b"ro.URI:SSK-RO:foo", None)),
1765	(20, UnknownNode(b"imm.URI:CHK:foo", None, deep_immutable=True)),
1766	(21, UnknownNode(None, b"URI:CHK:foo")),
1767	(22, UnknownNode(None, b"URI:CHK:foo", deep_immutable=True)),
1768	]
1769	ro_prefixed = [# These are valid, and the readcap should end up with a ro. prefix.
1770	(23, UnknownNode(None, b"foo")),
1771	(24, UnknownNode(None, b"ro.foo")),
1772	(25, UnknownNode(None, b"ro." + lit_uri)),
1773	(26, UnknownNode(b"bar", b"foo")),
1774	(27, UnknownNode(b"bar", b"ro.foo")),
1775	(28, UnknownNode(b"bar", b"ro." + lit_uri)),
1776	(29, UnknownNode(b"ro.foo", None)),
1777	(30, UnknownNode(b"ro." + lit_uri, None)),
1778	]
1779	imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1780	(31, UnknownNode(None, b"foo", deep_immutable=True)),
1781	(32, UnknownNode(None, b"ro.foo", deep_immutable=True)),
1782	(33, UnknownNode(None, b"imm.foo")),
1783	(34, UnknownNode(None, b"imm.foo", deep_immutable=True)),
1784	(35, UnknownNode(b"imm." + lit_uri, None)),
1785	(36, UnknownNode(b"imm." + lit_uri, None, deep_immutable=True)),
1786	(37, UnknownNode(None, b"imm." + lit_uri)),
1787	(38, UnknownNode(None, b"imm." + lit_uri, deep_immutable=True)),
1788	]
1789	error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1790	ok = ro_prefixed + imm_prefixed
1791
1792	for (i, n) in no_no + error + ok:
1793	self.failUnless(n.is_unknown(), i)
1794
1795	for (i, n) in no_no + error:
1796	self.failUnless(n.get_uri() is None, i)
1797	self.failUnless(n.get_write_uri() is None, i)
1798	self.failUnless(n.get_readonly_uri() is None, i)
1799
1800	for (i, n) in no_no + ok:
1801	n.raise_error()
1802
1803	for (i, n) in unknown_rw:
1804	self.failUnlessRaises(MustNotBeUnknownRWError, lambda n=n: n.raise_error())
1805
1806	for (i, n) in must_be_ro:
1807	self.failUnlessRaises(MustBeReadonlyError, lambda n=n: n.raise_error())
1808
1809	for (i, n) in must_be_imm:
1810	self.failUnlessRaises(MustBeDeepImmutableError, lambda n=n: n.raise_error())
1811
1812	for (i, n) in bad_uri:
1813	self.failUnlessRaises(uri.BadURIError, lambda n=n: n.raise_error())
1814
1815	for (i, n) in ok:
1816	self.failIf(n.get_readonly_uri() is None, i)
1817
1818	for (i, n) in ro_prefixed:
1819	self.failUnless(n.get_readonly_uri().startswith(b"ro."), i)
1820
1821	for (i, n) in imm_prefixed:
1822	self.failUnless(n.get_readonly_uri().startswith(b"imm."), i)
1823
1824
1825
1826	class DeepStats(testutil.ReallyEqualMixin, unittest.TestCase):
1827	def test_stats(self):
1828	ds = dirnode.DeepStats(None)
1829	ds.add("count-files")
1830	ds.add("size-immutable-files", 123)
1831	ds.histogram("size-files-histogram", 123)
1832	ds.max("largest-directory", 444)
1833
1834	s = ds.get_results()
1835	self.failUnlessReallyEqual(s["count-files"], 1)
1836	self.failUnlessReallyEqual(s["size-immutable-files"], 123)
1837	self.failUnlessReallyEqual(s["largest-directory"], 444)
1838	self.failUnlessReallyEqual(s["count-literal-files"], 0)
1839
1840	ds.add("count-files")
1841	ds.add("size-immutable-files", 321)
1842	ds.histogram("size-files-histogram", 321)
1843	ds.max("largest-directory", 2)
1844
1845	s = ds.get_results()
1846	self.failUnlessReallyEqual(s["count-files"], 2)
1847	self.failUnlessReallyEqual(s["size-immutable-files"], 444)
1848	self.failUnlessReallyEqual(s["largest-directory"], 444)
1849	self.failUnlessReallyEqual(s["count-literal-files"], 0)
1850	self.failUnlessReallyEqual(s["size-files-histogram"],
1851	[ (101, 316, 1), (317, 1000, 1) ])
1852
1853	ds = dirnode.DeepStats(None)
1854	for i in range(1, 1100):
1855	ds.histogram("size-files-histogram", i)
1856	ds.histogram("size-files-histogram", 41000100010001000) # 4TB
1857	s = ds.get_results()
1858	self.failUnlessReallyEqual(s["size-files-histogram"],
1859	[ (1, 3, 3),
1860	(4, 10, 7),
1861	(11, 31, 21),
1862	(32, 100, 69),
1863	(101, 316, 216),
1864	(317, 1000, 684),
1865	(1001, 3162, 99),
1866	(3162277660169, 10000000000000, 1),
1867	])
1868
1869	class UCWEingMutableFileNode(MutableFileNode):
1870	please_ucwe_after_next_upload = False
1871
1872	def _upload(self, new_contents, servermap):
1873	d = MutableFileNode._upload(self, new_contents, servermap)
1874	def _ucwe(res):
1875	if self.please_ucwe_after_next_upload:
1876	self.please_ucwe_after_next_upload = False
1877	raise UncoordinatedWriteError()
1878	return res
1879	d.addCallback(_ucwe)
1880	return d
1881
1882	class UCWEingNodeMaker(NodeMaker):
1883	def _create_mutable(self, cap):
1884	n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1885	self.default_encoding_parameters,
1886	self.history)
1887	return n.init_from_cap(cap)
1888
1889
1890	class Deleter(GridTestMixin, testutil.ReallyEqualMixin, unittest.TestCase):
1891	def test_retry(self):
1892	# ticket #550, a dirnode.delete which experiences an
1893	# UncoordinatedWriteError will fail with an incorrect "you're
1894	# deleting something which isn't there" NoSuchChildError exception.
1895
1896	# to trigger this, we start by creating a directory with a single
1897	# file in it. Then we create a special dirnode that uses a modified
1898	# MutableFileNode which will raise UncoordinatedWriteError once on
1899	# demand. We then call dirnode.delete, which ought to retry and
1900	# succeed.
1901
1902	self.basedir = self.mktemp()
1903	self.set_up_grid(oneshare=True)
1904	c0 = self.g.clients[0]
1905	d = c0.create_dirnode()
1906	small = upload.Data(b"Small enough for a LIT", None)
1907	def _created_dir(dn):
1908	self.root = dn
1909	self.root_uri = dn.get_uri()
1910	return dn.add_file(u"file", small)
1911	d.addCallback(_created_dir)
1912	def _do_delete(ignored):
1913	nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1914	c0.get_history(), c0.getServiceNamed("uploader"),
1915	c0.terminator,
1916	c0.get_encoding_parameters(),
1917	c0.mutable_file_default,
1918	c0._key_generator)
1919	n = nm.create_from_cap(self.root_uri)
1920	assert n._node.please_ucwe_after_next_upload == False
1921	n._node.please_ucwe_after_next_upload = True
1922	# This should succeed, not raise an exception
1923	return n.delete(u"file")
1924	d.addCallback(_do_delete)
1925
1926	return d
1927
1928	class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1929
1930	def test_overwrite(self):
1931	# note: This functionality could be tested without actually creating
1932	# several RSA keys. It would be faster without the GridTestMixin: use
1933	# dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1934	# of dn.add_file, and use a special NodeMaker that creates fake
1935	# mutable files.
1936	self.basedir = "dirnode/Adder/test_overwrite"
1937	self.set_up_grid(oneshare=True)
1938	c = self.g.clients[0]
1939	fileuri = make_chk_file_uri(1234)
1940	filenode = c.nodemaker.create_from_cap(fileuri)
1941	d = c.create_dirnode()
1942
1943	def _create_directory_tree(root_node):
1944	# Build
1945	# root/file1
1946	# root/file2
1947	# root/dir1
1948	d = root_node.add_file(u'file1', upload.Data(b"Important Things",
1949	None))
1950	d.addCallback(lambda res:
1951	root_node.add_file(u'file2', upload.Data(b"Sekrit Codes", None)))
1952	d.addCallback(lambda res:
1953	root_node.create_subdirectory(u"dir1"))
1954	d.addCallback(lambda res: root_node)
1955	return d
1956
1957	d.addCallback(_create_directory_tree)
1958
1959	def _test_adder(root_node):
1960	d = root_node.set_node(u'file1', filenode)
1961	# We've overwritten file1. Let's try it with a directory
1962	d.addCallback(lambda res:
1963	root_node.create_subdirectory(u'dir2'))
1964	d.addCallback(lambda res:
1965	root_node.set_node(u'dir2', filenode))
1966	# We try overwriting a file with a child while also specifying
1967	# overwrite=False. We should receive an ExistingChildError
1968	# when we do this.
1969	d.addCallback(lambda res:
1970	self.shouldFail(ExistingChildError, "set_node",
1971	"child 'file1' already exists",
1972	root_node.set_node, u"file1",
1973	filenode, overwrite=False))
1974	# If we try with a directory, we should see the same thing
1975	d.addCallback(lambda res:
1976	self.shouldFail(ExistingChildError, "set_node",
1977	"child 'dir1' already exists",
1978	root_node.set_node, u'dir1', filenode,
1979	overwrite=False))
1980	d.addCallback(lambda res:
1981	root_node.set_node(u'file1', filenode,
1982	overwrite=dirnode.ONLY_FILES))
1983	d.addCallback(lambda res:
1984	self.shouldFail(ExistingChildError, "set_node",
1985	"child 'dir1' already exists",
1986	root_node.set_node, u'dir1', filenode,
1987	overwrite=dirnode.ONLY_FILES))
1988	return d
1989
1990	d.addCallback(_test_adder)
1991	return d
1992
1993
1994	class DeterministicDirnode(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase):
1995	def setUp(self):
1996	# Copied from allmydata.test.mutable.test_filenode
1997	super(DeterministicDirnode, self).setUp()
1998	self._storage = FakeStorage()
1999	self._peers = list(
2000	make_peer(self._storage, n)
2001	for n
2002	in range(10)
2003	)
2004	self.nodemaker = make_nodemaker_with_peers(self._peers)
2005
2006	async def test_create_with_random_keypair(self):
2007	"""
2008	Create a dirnode using a random RSA keypair.
2009
2010	The writekey and fingerprint of the enclosed mutable filecap
2011	should match those derived from the given keypair.
2012	"""
2013	privkey, pubkey = create_signing_keypair(2048)
2014	writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
2015
2016	node = await self.nodemaker.create_new_mutable_directory(
2017	keypair=(pubkey, privkey)
2018	)
2019	self.failUnless(isinstance(node, dirnode.DirectoryNode))
2020
2021	dircap = uri.from_string(node.get_uri())
2022	self.failUnless(isinstance(dircap, uri.DirectoryURI))
2023
2024	filecap = dircap.get_filenode_cap()
2025	self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI))
2026
2027	self.failUnlessReallyEqual(filecap.writekey, writekey)
2028	self.failUnlessReallyEqual(filecap.fingerprint, fingerprint)
2029
2030	async def test_create_with_known_keypair(self):
2031	"""
2032	Create a dirnode using a known RSA keypair.
2033
2034	The writekey and fingerprint of the enclosed mutable filecap
2035	should match those derived from the given keypair. Because
2036	these values are derived deterministically, given the same
2037	keypair, the resulting filecap should also always be the same.
2038	"""
2039	# Generated with `openssl genrsa -out openssl-rsa-2048-2.txt 2048`
2040	pempath = FilePath(__file__).sibling("data").child("openssl-rsa-2048-2.txt")
2041	privkey = load_pem_private_key(pempath.getContent(), password=None)
2042	pubkey = privkey.public_key()
2043	writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
2044
2045	node = await self.nodemaker.create_new_mutable_directory(
2046	keypair=(pubkey, privkey)
2047	)
2048	self.failUnless(isinstance(node, dirnode.DirectoryNode))
2049
2050	dircap = uri.from_string(node.get_uri())
2051	self.failUnless(isinstance(dircap, uri.DirectoryURI))
2052
2053	filecap = dircap.get_filenode_cap()
2054	self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI))
2055
2056	self.failUnlessReallyEqual(filecap.writekey, writekey)
2057	self.failUnlessReallyEqual(filecap.fingerprint, fingerprint)
2058
2059	self.failUnlessReallyEqual(
2060	# Despite being named "to_string", this actually returns bytes..
2061	dircap.to_string(),
2062	b'URI:DIR2:n4opqgewgcn4mddu4oiippaxru:ukpe4z6xdlujdpguoabergyih3bj7iaafukdqzwthy2ytdd5bs2a'
2063	)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: