source: src/mlx/update.py@ 271:fa4c688d0fb4

Last change on this file since 271:fa4c688d0fb4 was 189:2d89178707a0, checked in by István Váradi <ivaradi@…>, 13 years ago

Added script to create an archive of the differences
File size: 19.6 KB
Line 
1# Module handling the download of updates
2
3#------------------------------------------------------------------------------
4
5from config import Config
6
7import os
8import sys
9import urllib2
10import tempfile
11import socket
12import subprocess
13import hashlib
14
15if os.name=="nt":
16 import win32api
17
18#------------------------------------------------------------------------------
19
20manifestName = "MLXMANIFEST"
21toremoveName = "toremove"
22
23#------------------------------------------------------------------------------
24
25class Manifest(object):
26 """The manifest of the files.
27
28 The manifest file consists of one line for each file. Each line contains 3
29 items separated by tabs:
30 - the path of the file relative to the distribution's root
31 - the size of the file
32 - the MD5 sum of the file."""
33 def __init__(self):
34 """Construct the manifest."""
35 self._files = {}
36
37 @property
38 def files(self):
39 """Get an iterator over the files.
40
41 Each file is returned as a 3-tuple with items as in the file."""
42 for (path, (size, sum)) in self._files.iteritems():
43 yield (path, size, sum)
44
45 def copy(self):
46 """Create a copy of the manifest."""
47 manifest = Manifest()
48 manifest._files = self._files.copy()
49 return manifest
50
51 def addFile(self, path, size, sum):
52 """Add a file to the manifest."""
53 self._files[path] = (size, sum)
54
55 def addFiles(self, baseDirectory, subdirectory):
56 """Add the files in the given directory and subdirectories of it to the
57 manifest."""
58 directory = baseDirectory
59 for d in subdirectory: directory = os.path.join(directory, d)
60
61 for entry in os.listdir(directory):
62 fullPath = os.path.join(directory, entry)
63 if os.path.isfile(fullPath):
64 size = os.stat(fullPath).st_size
65 sum = hashlib.md5()
66 with open(fullPath, "rb") as f:
67 while True:
68 data = f.read(4096)
69 if data: sum.update(data)
70 else: break
71 self.addFile("/".join(subdirectory + [entry]), size,
72 sum.hexdigest())
73 elif os.path.isdir(fullPath):
74 self.addFiles(baseDirectory, subdirectory + [entry])
75
76 def readFrom(self, file):
77 """Read a manifest from the given file object."""
78 for line in iter(file.readline, ""):
79 (path, size, sum) = line.strip().split("\t")
80 self._files[path] = (int(size), sum)
81
82 def writeInto(self, file):
83 """Write the manifest into the file at the given path."""
84 for (path, (size, sum)) in self._files.iteritems():
85 file.write("%s\t%d\t%s\n" % (path, size, sum))
86
87 def compare(self, other):
88 """Compare this manifest with the other one.
89
90 This returns a tuple of two lists:
91 - the files that are either different or are present in other, but not
92 here. Each file is returned as a 3-tuple as with other functions, the
93 size and sum are those of the new file.
94 - the paths of the files that are present here, but not in other."""
95 modifiedAndNew = []
96 for (path, otherSize, otherSum) in other.files:
97 if path in self._files:
98 (size, sum) = self._files[path]
99 if size!=otherSize or sum!=otherSum:
100 modifiedAndNew.append((path, otherSize, otherSum))
101 else:
102 modifiedAndNew.append((path, otherSize, otherSum))
103
104 removed = [path for path in self._files if path not in other._files]
105
106 return (modifiedAndNew, removed)
107
108 def __contains__(self, path):
109 """Determine if the given path is in the manifest."""
110 return path in self._files
111
112 def __getitem__(self, path):
113 """Get data of the file with the given path."""
114 return self._files[path] if path in self._files else None
115
116#------------------------------------------------------------------------------
117
118class ClientListener(object):
119 """A listener that sends any requests via a socket."""
120 def __init__(self, sock):
121 """Construct the listener."""
122 self._sock = sock
123
124 def downloadingManifest(self):
125 """Called when the downloading of the manifest has started."""
126 self._send(["downloadingManifest"])
127
128 def downloadedManifest(self):
129 """Called when the manifest has been downloaded."""
130 self._send(["downloadedManifest"])
131
132 def needSudo(self):
133 """Called when an admin-level program must be called to complete the
134 update.
135
136 This is not valid from a client, as that client is supposed to be the
137 admin-level program :)"""
138 assert False
139
140 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
141 numToRemoveLocal):
142 """Called when starting the downloading of the files."""
143 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
144 str(numToRemove), str(numToRemoveLocal)])
145
146 def setDownloaded(self, downloaded):
147 """Called periodically after downloading a certain amount of data."""
148 self._send(["setDownloaded", str(downloaded)])
149
150 def startRenaming(self):
151 """Called when the renaming of the downloaded files is started."""
152 self._send(["startRenaming"])
153
154 def renamed(self, path, count):
155 """Called when a file has been renamed."""
156 self._send(["renamed", path, str(count)])
157
158 def startRemoving(self):
159 """Called when the removing of files has started."""
160 self._send(["startRemoving"])
161
162 def removed(self, path, count):
163 """Called when a file has been removed."""
164 self._send(["removed", path, str(count)])
165
166 def writingManifest(self):
167 """Called when we have started writing the manifest."""
168 self._send(["writingManifest"])
169
170 def done(self):
171 """Called when the update has completed."""
172 self._send(["done"])
173
174 def failed(self, what):
175 """Called when something has failed."""
176 self._send(["failed", what])
177
178 def _send(self, words):
179 """Send the given words via the socket."""
180 self._sock.send("\t".join(words) + "\n")
181
182#------------------------------------------------------------------------------
183
184def readLocalManifest(directory):
185 """Read the local manifest from the given directory."""
186 manifestPath = os.path.join(directory, manifestName)
187
188 manifest = Manifest()
189 try:
190 with open(manifestPath, "rt") as f:
191 manifest.readFrom(f)
192 except Exception, e:
193 print "Error reading the manifest, ignoring:", str(e)
194 manifest = Manifest()
195
196 return manifest
197
198#------------------------------------------------------------------------------
199
200def prepareUpdate(directory, updateURL, listener):
201 """Prepare the update by downloading the manifest and comparing it with the
202 local one."""
203 manifest = readLocalManifest(directory)
204
205 updateURL += "/" + os.name
206
207 listener.downloadingManifest()
208 f = None
209 try:
210 updateManifest = Manifest()
211 f= urllib2.urlopen(updateURL + "/" + manifestName)
212 updateManifest.readFrom(f)
213
214 except Exception, e:
215 print >> sys.stderr, "Error downloading manifest:", str(e)
216 listener.failed(str(e))
217 return None
218 finally:
219 if f is not None: f.close()
220
221 listener.downloadedManifest()
222
223 (modifiedAndNew, removed) = manifest.compare(updateManifest)
224
225 return (manifest, updateManifest, modifiedAndNew, removed)
226
227#------------------------------------------------------------------------------
228
229def getToremoveFiles(directory):
230 """Add the files to remove from the toremove directory."""
231 toremove = []
232 toremoveDirectory = os.path.join(directory, toremoveName)
233 if os.path.isdir(toremoveDirectory):
234 for entry in os.listdir(toremoveDirectory):
235 toremove.append(os.path.join(toremoveName, entry))
236 return toremove
237
238#------------------------------------------------------------------------------
239
240def createLocalPath(directory, path):
241 """Create a local path from the given manifest path."""
242 localPath = directory
243 for element in path.split("/"):
244 localPath = os.path.join(localPath, element)
245 return localPath
246
247#------------------------------------------------------------------------------
248
249def getToremoveDir(toremoveDir, directory):
250 """Get the path of the directory that will contain the files that are to be
251 removed."""
252 if toremoveDir is None:
253 toremoveDir = os.path.join(directory, toremoveName)
254 try:
255 os.mkdir(toremoveDir)
256 except:
257 pass
258
259 return toremoveDir
260
261#------------------------------------------------------------------------------
262
263def removeFile(toremoveDir, directory, path):
264 """Remove the file at the given path or store it in a temporary directory.
265
266 If the removal of the file fails, it will be stored in a temporary
267 directory. This is useful for files thay may be open and cannot be removed
268 right away."""
269 try:
270 os.remove(path)
271 except:
272 try:
273 sum = hashlib.md5()
274 sum.update(path)
275 toremoveDir = getToremoveDir(toremoveDir, directory)
276 os.rename(path, os.path.join(toremoveDir, sum.hexdigest()))
277 except Exception, e:
278 print "Cannot remove file " + path + ": " + str(e)
279
280#------------------------------------------------------------------------------
281
282def updateFiles(directory, updateURL, listener,
283 manifest, modifiedAndNew, removed, localRemoved):
284 """Update the files according to the given information."""
285 totalSize = 0
286 for (path, size, sum) in modifiedAndNew:
287 totalSize += size
288
289 listener.setTotalSize(len(modifiedAndNew), totalSize,
290 len(removed), len(localRemoved))
291
292 downloaded = 0
293 fin = None
294 toremoveDir = None
295
296 try:
297 updateURL += "/" + os.name
298
299 for (path, size, sum) in modifiedAndNew:
300 targetFile = createLocalPath(directory, path)
301 targetFile += "."
302 targetFile += sum
303
304 targetDirectory = os.path.dirname(targetFile)
305 if not os.path.isdir(targetDirectory):
306 os.makedirs(targetDirectory)
307
308 with open(targetFile, "wb") as fout:
309 fin = urllib2.urlopen(updateURL + "/" + path)
310 while True:
311 data = fin.read(4096)
312 if not data:
313 break
314 fout.write(data)
315 downloaded += len(data)
316 listener.setDownloaded(downloaded)
317 fin.close()
318 fin = None
319
320 listener.startRenaming()
321 count = 0
322 for (path, size, sum) in modifiedAndNew:
323 targetFile = createLocalPath(directory, path)
324
325 downloadedFile = targetFile + "." + sum
326 if os.name=="nt" and os.path.isfile(targetFile):
327 removeFile(toremoveDir, directory, targetFile)
328 os.rename(downloadedFile, targetFile)
329 count += 1
330 listener.renamed(path, count)
331
332 listener.startRemoving()
333 count = 0
334 removed += localRemoved
335 removed.sort(reverse = True)
336 for path in removed:
337 removePath = createLocalPath(directory, path)
338 removeFile(toremoveDir, directory, removePath)
339
340 removeDirectory = os.path.dirname(removePath)
341 try:
342 os.removedirs(removeDirectory)
343 except:
344 pass
345
346 count += 1
347 listener.removed(path, count)
348
349 listener.writingManifest()
350
351 manifestPath = os.path.join(directory, manifestName)
352 with open(manifestPath, "wt") as f:
353 manifest.writeInto(f)
354
355 listener.done()
356 except Exception, e:
357 print >> sys.stderr, "Error:", str(e)
358 listener.failed(str(e))
359
360#------------------------------------------------------------------------------
361
362def isDirectoryWritable(directory):
363 """Determine if the given directory can be written."""
364 checkFile = os.path.join(directory, "writable.chk")
365 try:
366 f = open(checkFile, "wt")
367 f.close()
368 return True
369 except Exception, e:
370 return False
371 finally:
372 try:
373 os.remove(checkFile)
374 except:
375 pass
376
377#------------------------------------------------------------------------------
378
379def processMLXUpdate(buffer, listener):
380 """Process the given buffer supposedly containing a list of commands."""
381 endsWithLine = buffer[-1]=="\n"
382 lines = buffer.splitlines()
383
384 if endsWithLine:
385 buffer = ""
386 else:
387 buffer = lines[-1]
388 lines = lines[:-1]
389
390 for line in lines:
391 words = line.split("\t")
392 try:
393 command = words[0]
394 if command=="downloadingManifest":
395 listener.downloadingManifest()
396 elif command=="downloadedManifest":
397 listener.downloadedManifest()
398 elif command=="setTotalSize":
399 listener.setTotalSize(int(words[1]), long(words[2]),
400 int(words[3]), int(words[4]))
401 elif command=="setDownloaded":
402 listener.setDownloaded(long(words[1]))
403 elif command=="startRenaming":
404 listener.startRenaming()
405 elif command=="renamed":
406 listener.renamed(words[1], int(words[2]))
407 elif command=="startRemoving":
408 listener.startRemoving()
409 elif command=="removed":
410 listener.removed(words[1], int(words[2]))
411 elif command=="writingManifest":
412 listener.writingManifest()
413 elif command=="done":
414 listener.done()
415 elif command=="failed":
416 listener.failed(words[1])
417 except Exception, e:
418 print >> sys.stderr, "Failed to parse line '%s': %s" % \
419 (line, str(e))
420
421 return buffer
422
423#------------------------------------------------------------------------------
424
425def sudoUpdate(directory, updateURL, listener, manifest):
426 """Perform the update via the mlxupdate program."""
427 manifestFD = None
428 manifestFile = None
429 serverSocket = None
430 mlxUpdateSocket = None
431 try:
432 (manifestFD, manifestFile) = tempfile.mkstemp()
433 f = os.fdopen(manifestFD, "wt")
434 try:
435 manifest.writeInto(f)
436 finally:
437 f.close()
438 manifestFD = None
439
440 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
441 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
442 serverSocket.bind(("127.0.0.1", 0))
443 (_host, port) = serverSocket.getsockname()
444 serverSocket.listen(1)
445
446
447 if os.name=="nt":
448 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
449 str(port) + " " + manifestFile, "", 1)
450 else:
451 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
452 str(port), manifestFile],
453 shell = os.name=="nt")
454
455 (mlxUpdateSocket, _) = serverSocket.accept()
456 serverSocket.close()
457 serverSocket = None
458
459 buffer = ""
460 while True:
461 data = mlxUpdateSocket.recv(4096)
462 if not data:
463 break;
464
465 buffer += data
466 buffer = processMLXUpdate(buffer, listener)
467
468 mlxUpdateSocket.close()
469 mlxUpdateSocket = None
470
471 if os.name!="nt":
472 process.wait()
473
474 except Exception, e:
475 print >> sys.stderr, "Failed updating:", str(e)
476 listener.failed(str(e))
477 finally:
478 if serverSocket is not None:
479 try:
480 serverSocket.close()
481 except:
482 pass
483 if mlxUpdateSocket is not None:
484 try:
485 mlxUpdateSocket.close()
486 except:
487 pass
488 if manifestFD is not None:
489 try:
490 os.close(manifestFD)
491 except:
492 pass
493 if manifestFile is not None:
494 try:
495 os.remove(manifestFile)
496 except:
497 pass
498
499
500#------------------------------------------------------------------------------
501
502def update(directory, updateURL, listener, fromGUI = False):
503 """Perform the update."""
504 result = prepareUpdate(directory, updateURL, listener)
505 if result is None:
506 return
507
508 (manifest, updateManifest, modifiedAndNew, removed) = result
509 localRemoved = getToremoveFiles(directory)
510
511 if not modifiedAndNew and not removed and not localRemoved:
512 listener.done()
513 return
514
515 if fromGUI and not isDirectoryWritable(directory):
516 if listener.needSudo():
517 sudoUpdate(directory, updateURL, listener, updateManifest)
518 else:
519 updateFiles(directory, updateURL, listener, updateManifest,
520 modifiedAndNew, removed, localRemoved)
521
522#------------------------------------------------------------------------------
523
524def updateProcess():
525 """This is called in the child process, when we need a child process."""
526 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
527 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
528
529 config = Config()
530 config.load()
531
532 directory = os.path.dirname(sys.argv[0])
533
534 manifest = readLocalManifest(directory)
535
536 updateManifest = Manifest()
537 with open(sys.argv[2], "rt") as f:
538 updateManifest.readFrom(f)
539
540 (modifiedAndNew, removed) = manifest.compare(updateManifest)
541 localRemoved = getToremoveFiles(directory)
542
543 updateFiles(directory, config.updateURL,
544 ClientListener(clientSocket),
545 updateManifest, modifiedAndNew, removed, localRemoved)
546
547#------------------------------------------------------------------------------
548
549def buildManifest(directory):
550 """Build a manifest from the contents of the given directory, into the
551 given directory."""
552
553 manifestPath = os.path.join(directory, manifestName)
554 try:
555 os.remove(manifestPath)
556 except:
557 pass
558
559 manifest = Manifest()
560 manifest.addFiles(directory, [])
561 with open(manifestPath, "wt") as f:
562 manifest.writeInto(f)
563
564#------------------------------------------------------------------------------
565
566# if __name__ == "__main__":
567# manifest1 = Manifest()
568# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
569# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
570# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
571
572# with open("manifest1", "wt") as f:
573# manifest1.writeInto(f)
574
575# manifest2 = Manifest()
576# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
577# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
578# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
579
580# with open("manifest2", "wt") as f:
581# manifest2.writeInto(f)
582
583# manifest1 = Manifest()
584# with open("manifest1", "rt") as f:
585# manifest1.readFrom(f)
586
587# manifest2 = Manifest()
588# with open("manifest2", "rt") as f:
589# manifest2.readFrom(f)
590
591# (modifiedAndNew, removed) = manifest1.compare(manifest2)
592
593# for (path, size, sum) in modifiedAndNew:
594# print "modified or new:", path, size, sum
595
596# for path in removed:
597# print "removed:", path
598
599#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.