source: src/mlx/update.py@ 501:2fd9b3270f6d

Last change on this file since 501:2fd9b3270f6d was 401:15ad3c16ee11, checked in by István Váradi <ivaradi@…>, 12 years ago

The exception strings are converted from UTF-8 to unicode for proper logging (re #170)
File size: 20.5 KB
Line 
1
2from config import Config
3from util import utf2unicode
4
5import os
6import sys
7import urllib2
8import tempfile
9import socket
10import subprocess
11import hashlib
12
13if os.name=="nt":
14 import win32api
15
16#------------------------------------------------------------------------------
17
18## @package mlx.update
19#
20# Automatic update handling.
21#
22# The program can update itself automatically. For this purpose it maintains a
23# manifest of the files installed containing the relative paths, sizes and
24# checksums of each files. When the program starts up, this manifest file is
25# downloaded from the update server and is compared to the local one. Then the
26# updated and new files are downloaded with names that are created by appending
27# the checksum to the actual name, so as not to overwrite any existing files at
28# this stage. If all files are downloaded, the downloaded files are renamed to
29# their real names. On Windows, the old file is removed first to avoid trouble
30# with 'busy' files. If removing a file fails, the file will be moved to a
31# temporary directory, that will be removed when the program starts the next
32# time.
33
34#------------------------------------------------------------------------------
35
36manifestName = "MLXMANIFEST"
37toremoveName = "toremove"
38
39#------------------------------------------------------------------------------
40
41class Manifest(object):
42 """The manifest of the files.
43
44 The manifest file consists of one line for each file. Each line contains 3
45 items separated by tabs:
46 - the path of the file relative to the distribution's root
47 - the size of the file
48 - the MD5 sum of the file."""
49 def __init__(self):
50 """Construct the manifest."""
51 self._files = {}
52
53 @property
54 def files(self):
55 """Get an iterator over the files.
56
57 Each file is returned as a 3-tuple with items as in the file."""
58 for (path, (size, sum)) in self._files.iteritems():
59 yield (path, size, sum)
60
61 def copy(self):
62 """Create a copy of the manifest."""
63 manifest = Manifest()
64 manifest._files = self._files.copy()
65 return manifest
66
67 def addFile(self, path, size, sum):
68 """Add a file to the manifest."""
69 self._files[path] = (size, sum)
70
71 def addFiles(self, baseDirectory, subdirectory):
72 """Add the files in the given directory and subdirectories of it to the
73 manifest."""
74 directory = baseDirectory
75 for d in subdirectory: directory = os.path.join(directory, d)
76
77 for entry in os.listdir(directory):
78 fullPath = os.path.join(directory, entry)
79 if os.path.isfile(fullPath):
80 size = os.stat(fullPath).st_size
81 sum = hashlib.md5()
82 with open(fullPath, "rb") as f:
83 while True:
84 data = f.read(4096)
85 if data: sum.update(data)
86 else: break
87 self.addFile("/".join(subdirectory + [entry]), size,
88 sum.hexdigest())
89 elif os.path.isdir(fullPath):
90 self.addFiles(baseDirectory, subdirectory + [entry])
91
92 def readFrom(self, file):
93 """Read a manifest from the given file object."""
94 for line in iter(file.readline, ""):
95 (path, size, sum) = line.strip().split("\t")
96 self._files[path] = (int(size), sum)
97
98 def writeInto(self, file):
99 """Write the manifest into the file at the given path."""
100 for (path, (size, sum)) in self._files.iteritems():
101 file.write("%s\t%d\t%s\n" % (path, size, sum))
102
103 def compare(self, other):
104 """Compare this manifest with the other one.
105
106 This returns a tuple of two lists:
107 - the files that are either different or are present in other, but not
108 here. Each file is returned as a 3-tuple as with other functions, the
109 size and sum are those of the new file.
110 - the paths of the files that are present here, but not in other."""
111 modifiedAndNew = []
112 for (path, otherSize, otherSum) in other.files:
113 if path in self._files:
114 (size, sum) = self._files[path]
115 if size!=otherSize or sum!=otherSum:
116 modifiedAndNew.append((path, otherSize, otherSum))
117 else:
118 modifiedAndNew.append((path, otherSize, otherSum))
119
120 removed = [path for path in self._files if path not in other._files]
121
122 return (modifiedAndNew, removed)
123
124 def __contains__(self, path):
125 """Determine if the given path is in the manifest."""
126 return path in self._files
127
128 def __getitem__(self, path):
129 """Get data of the file with the given path."""
130 return self._files[path] if path in self._files else None
131
132#------------------------------------------------------------------------------
133
134class ClientListener(object):
135 """A listener that sends any requests via a socket."""
136 def __init__(self, sock):
137 """Construct the listener."""
138 self._sock = sock
139
140 def downloadingManifest(self):
141 """Called when the downloading of the manifest has started."""
142 self._send(["downloadingManifest"])
143
144 def downloadedManifest(self):
145 """Called when the manifest has been downloaded."""
146 self._send(["downloadedManifest"])
147
148 def needSudo(self):
149 """Called when an admin-level program must be called to complete the
150 update.
151
152 This is not valid from a client, as that client is supposed to be the
153 admin-level program :)"""
154 assert False
155
156 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
157 numToRemoveLocal):
158 """Called when starting the downloading of the files."""
159 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
160 str(numToRemove), str(numToRemoveLocal)])
161
162 def setDownloaded(self, downloaded):
163 """Called periodically after downloading a certain amount of data."""
164 self._send(["setDownloaded", str(downloaded)])
165
166 def startRenaming(self):
167 """Called when the renaming of the downloaded files is started."""
168 self._send(["startRenaming"])
169
170 def renamed(self, path, count):
171 """Called when a file has been renamed."""
172 self._send(["renamed", path, str(count)])
173
174 def startRemoving(self):
175 """Called when the removing of files has started."""
176 self._send(["startRemoving"])
177
178 def removed(self, path, count):
179 """Called when a file has been removed."""
180 self._send(["removed", path, str(count)])
181
182 def writingManifest(self):
183 """Called when we have started writing the manifest."""
184 self._send(["writingManifest"])
185
186 def done(self):
187 """Called when the update has completed."""
188 self._send(["done"])
189
190 def failed(self, what):
191 """Called when something has failed."""
192 self._send(["failed", what])
193
194 def _send(self, words):
195 """Send the given words via the socket."""
196 self._sock.send("\t".join(words) + "\n")
197
198#------------------------------------------------------------------------------
199
200def readLocalManifest(directory):
201 """Read the local manifest from the given directory."""
202 manifestPath = os.path.join(directory, manifestName)
203
204 manifest = Manifest()
205 try:
206 with open(manifestPath, "rt") as f:
207 manifest.readFrom(f)
208 except Exception, e:
209 print "Error reading the manifest, ignoring:", utf2unicode(str(e))
210 manifest = Manifest()
211
212 return manifest
213
214#------------------------------------------------------------------------------
215
216def prepareUpdate(directory, updateURL, listener):
217 """Prepare the update by downloading the manifest and comparing it with the
218 local one."""
219 manifest = readLocalManifest(directory)
220
221 updateURL += "/" + os.name
222
223 listener.downloadingManifest()
224 f = None
225 try:
226 updateManifest = Manifest()
227 f= urllib2.urlopen(updateURL + "/" + manifestName)
228 updateManifest.readFrom(f)
229
230 except Exception, e:
231 error = utf2unicode(str(e))
232 print >> sys.stderr, "Error downloading manifest:", error
233 listener.failed(error)
234 return None
235 finally:
236 if f is not None: f.close()
237
238 listener.downloadedManifest()
239
240 (modifiedAndNew, removed) = manifest.compare(updateManifest)
241
242 return (manifest, updateManifest, modifiedAndNew, removed)
243
244#------------------------------------------------------------------------------
245
246def getToremoveFiles(directory):
247 """Add the files to remove from the toremove directory."""
248 toremove = []
249 toremoveDirectory = os.path.join(directory, toremoveName)
250 if os.path.isdir(toremoveDirectory):
251 for entry in os.listdir(toremoveDirectory):
252 toremove.append(os.path.join(toremoveName, entry))
253 return toremove
254
255#------------------------------------------------------------------------------
256
257def createLocalPath(directory, path):
258 """Create a local path from the given manifest path."""
259 localPath = directory
260 for element in path.split("/"):
261 localPath = os.path.join(localPath, element)
262 return localPath
263
264#------------------------------------------------------------------------------
265
266def getToremoveDir(toremoveDir, directory):
267 """Get the path of the directory that will contain the files that are to be
268 removed."""
269 if toremoveDir is None:
270 toremoveDir = os.path.join(directory, toremoveName)
271 try:
272 os.mkdir(toremoveDir)
273 except:
274 pass
275
276 return toremoveDir
277
278#------------------------------------------------------------------------------
279
280def removeFile(toremoveDir, directory, path):
281 """Remove the file at the given path or store it in a temporary directory.
282
283 If the removal of the file fails, it will be stored in a temporary
284 directory. This is useful for files thay may be open and cannot be removed
285 right away."""
286 try:
287 os.remove(path)
288 except:
289 try:
290 sum = hashlib.md5()
291 sum.update(path)
292 toremoveDir = getToremoveDir(toremoveDir, directory)
293 os.rename(path, os.path.join(toremoveDir, sum.hexdigest()))
294 except Exception, e:
295 print "Cannot remove file " + path + ": " + utf2unicode(str(e))
296
297#------------------------------------------------------------------------------
298
299def updateFiles(directory, updateURL, listener,
300 manifest, modifiedAndNew, removed, localRemoved):
301 """Update the files according to the given information."""
302 totalSize = 0
303 for (path, size, sum) in modifiedAndNew:
304 totalSize += size
305
306 listener.setTotalSize(len(modifiedAndNew), totalSize,
307 len(removed), len(localRemoved))
308
309 downloaded = 0
310 fin = None
311 toremoveDir = None
312
313 try:
314 updateURL += "/" + os.name
315
316 for (path, size, sum) in modifiedAndNew:
317 targetFile = createLocalPath(directory, path)
318 targetFile += "."
319 targetFile += sum
320
321 targetDirectory = os.path.dirname(targetFile)
322 if not os.path.isdir(targetDirectory):
323 os.makedirs(targetDirectory)
324
325 with open(targetFile, "wb") as fout:
326 fin = urllib2.urlopen(updateURL + "/" + path)
327 while True:
328 data = fin.read(4096)
329 if not data:
330 break
331 fout.write(data)
332 downloaded += len(data)
333 listener.setDownloaded(downloaded)
334 fin.close()
335 fin = None
336
337 listener.startRenaming()
338 count = 0
339 for (path, size, sum) in modifiedAndNew:
340 targetFile = createLocalPath(directory, path)
341
342 downloadedFile = targetFile + "." + sum
343 if os.name=="nt" and os.path.isfile(targetFile):
344 removeFile(toremoveDir, directory, targetFile)
345 os.rename(downloadedFile, targetFile)
346 count += 1
347 listener.renamed(path, count)
348
349 listener.startRemoving()
350 count = 0
351 removed += localRemoved
352 removed.sort(reverse = True)
353 for path in removed:
354 removePath = createLocalPath(directory, path)
355 removeFile(toremoveDir, directory, removePath)
356
357 removeDirectory = os.path.dirname(removePath)
358 try:
359 os.removedirs(removeDirectory)
360 except:
361 pass
362
363 count += 1
364 listener.removed(path, count)
365
366 listener.writingManifest()
367
368 manifestPath = os.path.join(directory, manifestName)
369 with open(manifestPath, "wt") as f:
370 manifest.writeInto(f)
371
372 listener.done()
373 except Exception, e:
374 error = utf2unicode(str(e))
375 print >> sys.stderr, "Error:", error
376 listener.failed(error)
377
378#------------------------------------------------------------------------------
379
380def isDirectoryWritable(directory):
381 """Determine if the given directory can be written."""
382 checkFile = os.path.join(directory, "writable.chk")
383 try:
384 f = open(checkFile, "wt")
385 f.close()
386 return True
387 except Exception, e:
388 return False
389 finally:
390 try:
391 os.remove(checkFile)
392 except:
393 pass
394
395#------------------------------------------------------------------------------
396
397def processMLXUpdate(buffer, listener):
398 """Process the given buffer supposedly containing a list of commands."""
399 endsWithLine = buffer[-1]=="\n"
400 lines = buffer.splitlines()
401
402 if endsWithLine:
403 buffer = ""
404 else:
405 buffer = lines[-1]
406 lines = lines[:-1]
407
408 for line in lines:
409 words = line.split("\t")
410 try:
411 command = words[0]
412 if command=="downloadingManifest":
413 listener.downloadingManifest()
414 elif command=="downloadedManifest":
415 listener.downloadedManifest()
416 elif command=="setTotalSize":
417 listener.setTotalSize(int(words[1]), long(words[2]),
418 int(words[3]), int(words[4]))
419 elif command=="setDownloaded":
420 listener.setDownloaded(long(words[1]))
421 elif command=="startRenaming":
422 listener.startRenaming()
423 elif command=="renamed":
424 listener.renamed(words[1], int(words[2]))
425 elif command=="startRemoving":
426 listener.startRemoving()
427 elif command=="removed":
428 listener.removed(words[1], int(words[2]))
429 elif command=="writingManifest":
430 listener.writingManifest()
431 elif command=="done":
432 listener.done()
433 elif command=="failed":
434 listener.failed(words[1])
435 except Exception, e:
436 print >> sys.stderr, "Failed to parse line '%s': %s" % \
437 (line, utf2unicode(str(e)))
438
439 return buffer
440
441#------------------------------------------------------------------------------
442
443def sudoUpdate(directory, updateURL, listener, manifest):
444 """Perform the update via the mlxupdate program."""
445 manifestFD = None
446 manifestFile = None
447 serverSocket = None
448 mlxUpdateSocket = None
449 try:
450 (manifestFD, manifestFile) = tempfile.mkstemp()
451 f = os.fdopen(manifestFD, "wt")
452 try:
453 manifest.writeInto(f)
454 finally:
455 f.close()
456 manifestFD = None
457
458 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
459 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
460 serverSocket.bind(("127.0.0.1", 0))
461 (_host, port) = serverSocket.getsockname()
462 serverSocket.listen(1)
463
464
465 if os.name=="nt":
466 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
467 str(port) + " " + manifestFile, "", 1)
468 else:
469 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
470 str(port), manifestFile],
471 shell = os.name=="nt")
472
473 (mlxUpdateSocket, _) = serverSocket.accept()
474 serverSocket.close()
475 serverSocket = None
476
477 buffer = ""
478 while True:
479 data = mlxUpdateSocket.recv(4096)
480 if not data:
481 break;
482
483 buffer += data
484 buffer = processMLXUpdate(buffer, listener)
485
486 mlxUpdateSocket.close()
487 mlxUpdateSocket = None
488
489 if os.name!="nt":
490 process.wait()
491
492 except Exception, e:
493 error = utf2unicode(str(e))
494 print >> sys.stderr, "Failed updating:", error
495 listener.failed(error)
496 finally:
497 if serverSocket is not None:
498 try:
499 serverSocket.close()
500 except:
501 pass
502 if mlxUpdateSocket is not None:
503 try:
504 mlxUpdateSocket.close()
505 except:
506 pass
507 if manifestFD is not None:
508 try:
509 os.close(manifestFD)
510 except:
511 pass
512 if manifestFile is not None:
513 try:
514 os.remove(manifestFile)
515 except:
516 pass
517
518
519#------------------------------------------------------------------------------
520
521def update(directory, updateURL, listener, fromGUI = False):
522 """Perform the update."""
523 result = prepareUpdate(directory, updateURL, listener)
524 if result is None:
525 return
526
527 (manifest, updateManifest, modifiedAndNew, removed) = result
528 localRemoved = getToremoveFiles(directory)
529
530 if not modifiedAndNew and not removed and not localRemoved:
531 listener.done()
532 return
533
534 if fromGUI and not isDirectoryWritable(directory):
535 if listener.needSudo():
536 sudoUpdate(directory, updateURL, listener, updateManifest)
537 else:
538 updateFiles(directory, updateURL, listener, updateManifest,
539 modifiedAndNew, removed, localRemoved)
540
541#------------------------------------------------------------------------------
542
543def updateProcess():
544 """This is called in the child process, when we need a child process."""
545 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
546 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
547
548 config = Config()
549 config.load()
550
551 directory = os.path.dirname(sys.argv[0])
552
553 manifest = readLocalManifest(directory)
554
555 updateManifest = Manifest()
556 with open(sys.argv[2], "rt") as f:
557 updateManifest.readFrom(f)
558
559 (modifiedAndNew, removed) = manifest.compare(updateManifest)
560 localRemoved = getToremoveFiles(directory)
561
562 updateFiles(directory, config.updateURL,
563 ClientListener(clientSocket),
564 updateManifest, modifiedAndNew, removed, localRemoved)
565
566#------------------------------------------------------------------------------
567
568def buildManifest(directory):
569 """Build a manifest from the contents of the given directory, into the
570 given directory."""
571
572 manifestPath = os.path.join(directory, manifestName)
573 try:
574 os.remove(manifestPath)
575 except:
576 pass
577
578 manifest = Manifest()
579 manifest.addFiles(directory, [])
580 with open(manifestPath, "wt") as f:
581 manifest.writeInto(f)
582
583#------------------------------------------------------------------------------
584
585# if __name__ == "__main__":
586# manifest1 = Manifest()
587# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
588# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
589# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
590
591# with open("manifest1", "wt") as f:
592# manifest1.writeInto(f)
593
594# manifest2 = Manifest()
595# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
596# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
597# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
598
599# with open("manifest2", "wt") as f:
600# manifest2.writeInto(f)
601
602# manifest1 = Manifest()
603# with open("manifest1", "rt") as f:
604# manifest1.readFrom(f)
605
606# manifest2 = Manifest()
607# with open("manifest2", "rt") as f:
608# manifest2.readFrom(f)
609
610# (modifiedAndNew, removed) = manifest1.compare(manifest2)
611
612# for (path, size, sum) in modifiedAndNew:
613# print "modified or new:", path, size, sum
614
615# for path in removed:
616# print "removed:", path
617
618#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.