source: src/mlx/update.py@ 795:e6182320ab12

Last change on this file since 795:e6182320ab12 was 788:805d2ae4d025, checked in by István Váradi <ivaradi@…>, 8 years ago

Made the update procedure more resilient to errors
File size: 21.5 KB
Line 
1
2from config import Config
3from util import utf2unicode
4
5import os
6import sys
7import urllib2
8import tempfile
9import socket
10import subprocess
11import hashlib
12import traceback
13
14if os.name=="nt":
15 import win32api
16
17#------------------------------------------------------------------------------
18
19## @package mlx.update
20#
21# Automatic update handling.
22#
23# The program can update itself automatically. For this purpose it maintains a
24# manifest of the files installed containing the relative paths, sizes and
25# checksums of each files. When the program starts up, this manifest file is
26# downloaded from the update server and is compared to the local one. Then the
27# updated and new files are downloaded with names that are created by appending
28# the checksum to the actual name, so as not to overwrite any existing files at
29# this stage. If all files are downloaded, the downloaded files are renamed to
30# their real names. On Windows, the old file is removed first to avoid trouble
31# with 'busy' files. If removing a file fails, the file will be moved to a
32# temporary directory, that will be removed when the program starts the next
33# time.
34
35#------------------------------------------------------------------------------
36
37manifestName = "MLXMANIFEST"
38toremoveName = "toremove"
39
40#------------------------------------------------------------------------------
41
42class Manifest(object):
43 """The manifest of the files.
44
45 The manifest file consists of one line for each file. Each line contains 3
46 items separated by tabs:
47 - the path of the file relative to the distribution's root
48 - the size of the file
49 - the MD5 sum of the file."""
50 def __init__(self):
51 """Construct the manifest."""
52 self._files = {}
53
54 @property
55 def files(self):
56 """Get an iterator over the files.
57
58 Each file is returned as a 3-tuple with items as in the file."""
59 for (path, (size, sum)) in self._files.iteritems():
60 yield (path, size, sum)
61
62 def copy(self):
63 """Create a copy of the manifest."""
64 manifest = Manifest()
65 manifest._files = self._files.copy()
66 return manifest
67
68 def addFile(self, path, size, sum):
69 """Add a file to the manifest."""
70 self._files[path] = (size, sum)
71
72 def addFiles(self, baseDirectory, subdirectory):
73 """Add the files in the given directory and subdirectories of it to the
74 manifest."""
75 directory = baseDirectory
76 for d in subdirectory: directory = os.path.join(directory, d)
77
78 for entry in os.listdir(directory):
79 fullPath = os.path.join(directory, entry)
80 if os.path.isfile(fullPath):
81 size = os.stat(fullPath).st_size
82 sum = hashlib.md5()
83 with open(fullPath, "rb") as f:
84 while True:
85 data = f.read(4096)
86 if data: sum.update(data)
87 else: break
88 self.addFile("/".join(subdirectory + [entry]), size,
89 sum.hexdigest())
90 elif os.path.isdir(fullPath):
91 self.addFiles(baseDirectory, subdirectory + [entry])
92
93 def readFrom(self, file):
94 """Read a manifest from the given file object."""
95 for line in iter(file.readline, ""):
96 (path, size, sum) = line.strip().split("\t")
97 self._files[path] = (int(size), sum)
98
99 def writeInto(self, file):
100 """Write the manifest into the file at the given path."""
101 for (path, (size, sum)) in self._files.iteritems():
102 file.write("%s\t%d\t%s\n" % (path, size, sum))
103
104 def compare(self, other):
105 """Compare this manifest with the other one.
106
107 This returns a tuple of two lists:
108 - the files that are either different or are present in other, but not
109 here. Each file is returned as a 3-tuple as with other functions, the
110 size and sum are those of the new file.
111 - the paths of the files that are present here, but not in other."""
112 modifiedAndNew = []
113 for (path, otherSize, otherSum) in other.files:
114 if path in self._files:
115 (size, sum) = self._files[path]
116 if size!=otherSize or sum!=otherSum:
117 modifiedAndNew.append((path, otherSize, otherSum))
118 else:
119 modifiedAndNew.append((path, otherSize, otherSum))
120
121 removed = [path for path in self._files if path not in other._files]
122
123 return (modifiedAndNew, removed)
124
125 def __contains__(self, path):
126 """Determine if the given path is in the manifest."""
127 return path in self._files
128
129 def __getitem__(self, path):
130 """Get data of the file with the given path."""
131 return self._files[path] if path in self._files else None
132
133#------------------------------------------------------------------------------
134
135class ClientListener(object):
136 """A listener that sends any requests via a socket."""
137 def __init__(self, sock):
138 """Construct the listener."""
139 self._sock = sock
140
141 def downloadingManifest(self):
142 """Called when the downloading of the manifest has started."""
143 self._send(["downloadingManifest"])
144
145 def downloadedManifest(self):
146 """Called when the manifest has been downloaded."""
147 self._send(["downloadedManifest"])
148
149 def needSudo(self):
150 """Called when an admin-level program must be called to complete the
151 update.
152
153 This is not valid from a client, as that client is supposed to be the
154 admin-level program :)"""
155 assert False
156
157 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
158 numToRemoveLocal):
159 """Called when starting the downloading of the files."""
160 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
161 str(numToRemove), str(numToRemoveLocal)])
162
163 def setDownloaded(self, downloaded):
164 """Called periodically after downloading a certain amount of data."""
165 self._send(["setDownloaded", str(downloaded)])
166
167 def startRenaming(self):
168 """Called when the renaming of the downloaded files is started."""
169 self._send(["startRenaming"])
170
171 def renamed(self, path, count):
172 """Called when a file has been renamed."""
173 self._send(["renamed", path, str(count)])
174
175 def startRemoving(self):
176 """Called when the removing of files has started."""
177 self._send(["startRemoving"])
178
179 def removed(self, path, count):
180 """Called when a file has been removed."""
181 self._send(["removed", path, str(count)])
182
183 def writingManifest(self):
184 """Called when we have started writing the manifest."""
185 self._send(["writingManifest"])
186
187 def done(self):
188 """Called when the update has completed."""
189 self._send(["done"])
190
191 def failed(self, what):
192 """Called when something has failed."""
193 self._send(["failed", what])
194
195 def _send(self, words):
196 """Send the given words via the socket."""
197 self._sock.send("\t".join(words) + "\n")
198
199#------------------------------------------------------------------------------
200
201def readLocalManifest(directory):
202 """Read the local manifest from the given directory."""
203 manifestPath = os.path.join(directory, manifestName)
204
205 manifest = Manifest()
206 try:
207 with open(manifestPath, "rt") as f:
208 manifest.readFrom(f)
209 except Exception, e:
210 print "Error reading the manifest, ignoring:", utf2unicode(str(e))
211 manifest = Manifest()
212
213 return manifest
214
215#------------------------------------------------------------------------------
216
217def prepareUpdate(directory, updateURL, listener):
218 """Prepare the update by downloading the manifest and comparing it with the
219 local one."""
220 manifest = readLocalManifest(directory)
221
222 updateURL += "/" + os.name
223
224 listener.downloadingManifest()
225 f = None
226 try:
227 updateManifest = Manifest()
228 f= urllib2.urlopen(updateURL + "/" + manifestName)
229 updateManifest.readFrom(f)
230
231 except Exception, e:
232 error = utf2unicode(str(e))
233 print >> sys.stderr, "Error downloading manifest:", error
234 listener.failed(error)
235 return None
236 finally:
237 if f is not None: f.close()
238
239 listener.downloadedManifest()
240
241 (modifiedAndNew, removed) = manifest.compare(updateManifest)
242
243 return (manifest, updateManifest, modifiedAndNew, removed)
244
245#------------------------------------------------------------------------------
246
247def getToremoveFiles(directory):
248 """Add the files to remove from the toremove directory."""
249 toremove = []
250 toremoveDirectory = os.path.join(directory, toremoveName)
251 if os.path.isdir(toremoveDirectory):
252 for entry in os.listdir(toremoveDirectory):
253 toremove.append(os.path.join(toremoveName, entry))
254 return toremove
255
256#------------------------------------------------------------------------------
257
258def createLocalPath(directory, path):
259 """Create a local path from the given manifest path."""
260 localPath = directory
261 for element in path.split("/"):
262 localPath = os.path.join(localPath, element)
263 return localPath
264
265#------------------------------------------------------------------------------
266
267def getToremoveDir(toremoveDir, directory):
268 """Get the path of the directory that will contain the files that are to be
269 removed."""
270 if toremoveDir is None:
271 toremoveDir = os.path.join(directory, toremoveName)
272 try:
273 os.mkdir(toremoveDir)
274 except:
275 pass
276
277 return toremoveDir
278
279#------------------------------------------------------------------------------
280
281def removeFile(toremoveDir, directory, path):
282 """Remove the file at the given path or store it in a temporary directory.
283
284 If the removal of the file fails, it will be stored in a temporary
285 directory. This is useful for files thay may be open and cannot be removed
286 right away."""
287 try:
288 os.remove(path)
289 except:
290 try:
291 sum = hashlib.md5()
292 sum.update(path)
293 toremoveDir = getToremoveDir(toremoveDir, directory)
294 targetPath = os.path.join(toremoveDir, sum.hexdigest())
295 try:
296 os.remove(targetPath)
297 except:
298 pass
299 os.rename(path, targetPath)
300 except Exception, e:
301 print "Cannot remove file " + path + ": " + utf2unicode(str(e))
302
303#------------------------------------------------------------------------------
304
305def removeFiles(directory, listener, removed, count):
306 """Remove the given files."""
307 toremoveDir = None
308
309 listener.startRemoving()
310
311 removed.sort(reverse = True)
312 for path in removed:
313 removePath = createLocalPath(directory, path)
314 removeFile(toremoveDir, directory, removePath)
315
316 removeDirectory = os.path.dirname(removePath)
317 try:
318 os.removedirs(removeDirectory)
319 except:
320 pass
321
322 count += 1
323 listener.removed(path, count)
324
325 return count
326
327#------------------------------------------------------------------------------
328
329def updateFiles(directory, updateURL, listener,
330 manifest, modifiedAndNew, removed, localRemoved):
331 """Update the files according to the given information."""
332 totalSize = 0
333 for (path, size, sum) in modifiedAndNew:
334 totalSize += size
335
336 listener.setTotalSize(len(modifiedAndNew), totalSize,
337 len(removed), len(localRemoved))
338
339 downloaded = 0
340 fin = None
341 toremoveDir = None
342
343 try:
344 updateURL += "/" + os.name
345
346 removeCount = 0
347 if localRemoved:
348 removeCount = removeFiles(directory, listener,
349 localRemoved, removeCount)
350
351 for (path, size, sum) in modifiedAndNew:
352 targetFile = createLocalPath(directory, path)
353 targetFile += "."
354 targetFile += sum
355
356 targetDirectory = os.path.dirname(targetFile)
357 if not os.path.isdir(targetDirectory):
358 os.makedirs(targetDirectory)
359
360 with open(targetFile, "wb") as fout:
361 fin = urllib2.urlopen(updateURL + "/" + path)
362 while True:
363 data = fin.read(4096)
364 if not data:
365 break
366 fout.write(data)
367 downloaded += len(data)
368 listener.setDownloaded(downloaded)
369 fin.close()
370 fin = None
371
372 listener.startRenaming()
373 count = 0
374 for (path, size, sum) in modifiedAndNew:
375 targetFile = createLocalPath(directory, path)
376
377 downloadedFile = targetFile + "." + sum
378 if os.name=="nt" and os.path.isfile(targetFile):
379 removeFile(toremoveDir, directory, targetFile)
380 os.rename(downloadedFile, targetFile)
381 count += 1
382 listener.renamed(path, count)
383
384 removeFiles(directory, listener, removed, removeCount)
385
386 listener.writingManifest()
387
388 manifestPath = os.path.join(directory, manifestName)
389 with open(manifestPath, "wt") as f:
390 manifest.writeInto(f)
391
392 listener.done()
393 except Exception, e:
394 exc = traceback.format_exc()
395 print >> sys.stderr, utf2unicode(exc)
396
397 error = utf2unicode(str(e))
398 print >> sys.stderr, "Error:", error
399
400 listener.failed(error)
401
402#------------------------------------------------------------------------------
403
404def isDirectoryWritable(directory):
405 """Determine if the given directory can be written."""
406 checkFile = os.path.join(directory, "writable.chk")
407 try:
408 f = open(checkFile, "wt")
409 f.close()
410 return True
411 except Exception, e:
412 return False
413 finally:
414 try:
415 os.remove(checkFile)
416 except:
417 pass
418
419#------------------------------------------------------------------------------
420
421def processMLXUpdate(buffer, listener):
422 """Process the given buffer supposedly containing a list of commands."""
423 endsWithLine = buffer[-1]=="\n"
424 lines = buffer.splitlines()
425
426 if endsWithLine:
427 buffer = ""
428 else:
429 buffer = lines[-1]
430 lines = lines[:-1]
431
432 for line in lines:
433 words = line.split("\t")
434 try:
435 command = words[0]
436 if command=="downloadingManifest":
437 listener.downloadingManifest()
438 elif command=="downloadedManifest":
439 listener.downloadedManifest()
440 elif command=="setTotalSize":
441 listener.setTotalSize(int(words[1]), long(words[2]),
442 int(words[3]), int(words[4]))
443 elif command=="setDownloaded":
444 listener.setDownloaded(long(words[1]))
445 elif command=="startRenaming":
446 listener.startRenaming()
447 elif command=="renamed":
448 listener.renamed(words[1], int(words[2]))
449 elif command=="startRemoving":
450 listener.startRemoving()
451 elif command=="removed":
452 listener.removed(words[1], int(words[2]))
453 elif command=="writingManifest":
454 listener.writingManifest()
455 elif command=="done":
456 listener.done()
457 elif command=="failed":
458 listener.failed(words[1])
459 except Exception, e:
460 print >> sys.stderr, "Failed to parse line '%s': %s" % \
461 (line, utf2unicode(str(e)))
462
463 return buffer
464
465#------------------------------------------------------------------------------
466
467def sudoUpdate(directory, updateURL, listener, manifest):
468 """Perform the update via the mlxupdate program."""
469 manifestFD = None
470 manifestFile = None
471 serverSocket = None
472 mlxUpdateSocket = None
473 try:
474 (manifestFD, manifestFile) = tempfile.mkstemp()
475 f = os.fdopen(manifestFD, "wt")
476 try:
477 manifest.writeInto(f)
478 finally:
479 f.close()
480 manifestFD = None
481
482 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
483 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
484 serverSocket.bind(("127.0.0.1", 0))
485 (_host, port) = serverSocket.getsockname()
486 serverSocket.listen(1)
487
488
489 if os.name=="nt":
490 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
491 str(port) + " " + manifestFile + " " + updateURL, "", 1)
492 else:
493 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
494 str(port), manifestFile, updateURL],
495 shell = os.name=="nt")
496
497 (mlxUpdateSocket, _) = serverSocket.accept()
498 serverSocket.close()
499 serverSocket = None
500
501 buffer = ""
502 while True:
503 data = mlxUpdateSocket.recv(4096)
504 if not data:
505 break;
506
507 buffer += data
508 buffer = processMLXUpdate(buffer, listener)
509
510 mlxUpdateSocket.close()
511 mlxUpdateSocket = None
512
513 if os.name!="nt":
514 process.wait()
515
516 except Exception, e:
517 error = utf2unicode(str(e))
518 print >> sys.stderr, "Failed updating:", error
519 listener.failed(error)
520 finally:
521 if serverSocket is not None:
522 try:
523 serverSocket.close()
524 except:
525 pass
526 if mlxUpdateSocket is not None:
527 try:
528 mlxUpdateSocket.close()
529 except:
530 pass
531 if manifestFD is not None:
532 try:
533 os.close(manifestFD)
534 except:
535 pass
536 if manifestFile is not None:
537 try:
538 os.remove(manifestFile)
539 except:
540 pass
541
542
543#------------------------------------------------------------------------------
544
545def update(directory, updateURL, listener, fromGUI = False):
546 """Perform the update."""
547 try:
548 result = prepareUpdate(directory, updateURL, listener)
549 if result is None:
550 return
551
552 (manifest, updateManifest, modifiedAndNew, removed) = result
553 localRemoved = getToremoveFiles(directory)
554
555 if not modifiedAndNew and not removed and not localRemoved:
556 listener.done()
557 return
558
559 if fromGUI and not isDirectoryWritable(directory):
560 if listener.needSudo():
561 sudoUpdate(directory, updateURL, listener, updateManifest)
562 else:
563 updateFiles(directory, updateURL, listener, updateManifest,
564 modifiedAndNew, removed, localRemoved)
565 except Exception, e:
566 exc = traceback.format_exc()
567 print >> sys.stderr, utf2unicode(exc)
568
569 error = utf2unicode(str(e))
570 print >> sys.stderr, "Update error:", error
571
572 listener.failed(error)
573
574#------------------------------------------------------------------------------
575
576def updateProcess():
577 """This is called in the child process, when we need a child process."""
578 try:
579 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
580 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
581
582 directory = os.path.dirname(sys.argv[0])
583
584 manifest = readLocalManifest(directory)
585
586 updateManifest = Manifest()
587 with open(sys.argv[2], "rt") as f:
588 updateManifest.readFrom(f)
589
590 (modifiedAndNew, removed) = manifest.compare(updateManifest)
591 localRemoved = getToremoveFiles(directory)
592
593 updateFiles(directory, sys.argv[3],
594 ClientListener(clientSocket),
595 updateManifest, modifiedAndNew, removed, localRemoved)
596 except:
597 exc = traceback.format_exc()
598 print >> sys.stderr, utf2unicode(exc)
599
600#------------------------------------------------------------------------------
601
602def buildManifest(directory):
603 """Build a manifest from the contents of the given directory, into the
604 given directory."""
605
606 manifestPath = os.path.join(directory, manifestName)
607 try:
608 os.remove(manifestPath)
609 except:
610 pass
611
612 manifest = Manifest()
613 manifest.addFiles(directory, [])
614 with open(manifestPath, "wt") as f:
615 manifest.writeInto(f)
616
617#------------------------------------------------------------------------------
618
619# if __name__ == "__main__":
620# manifest1 = Manifest()
621# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
622# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
623# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
624
625# with open("manifest1", "wt") as f:
626# manifest1.writeInto(f)
627
628# manifest2 = Manifest()
629# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
630# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
631# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
632
633# with open("manifest2", "wt") as f:
634# manifest2.writeInto(f)
635
636# manifest1 = Manifest()
637# with open("manifest1", "rt") as f:
638# manifest1.readFrom(f)
639
640# manifest2 = Manifest()
641# with open("manifest2", "rt") as f:
642# manifest2.readFrom(f)
643
644# (modifiedAndNew, removed) = manifest1.compare(manifest2)
645
646# for (path, size, sum) in modifiedAndNew:
647# print "modified or new:", path, size, sum
648
649# for path in removed:
650# print "removed:", path
651
652#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.