source: src/mlx/update.py@ 385:f775fd26f011

Last change on this file since 385:f775fd26f011 was 298:24c67ec5cdca, checked in by István Váradi <ivaradi@…>, 12 years ago

Documented the non-GUI modules
File size: 20.4 KB
Line 
1
2from config import Config
3
4import os
5import sys
6import urllib2
7import tempfile
8import socket
9import subprocess
10import hashlib
11
12if os.name=="nt":
13 import win32api
14
15#------------------------------------------------------------------------------
16
17## @package mlx.update
18#
19# Automatic update handling.
20#
21# The program can update itself automatically. For this purpose it maintains a
22# manifest of the files installed containing the relative paths, sizes and
23# checksums of each files. When the program starts up, this manifest file is
24# downloaded from the update server and is compared to the local one. Then the
25# updated and new files are downloaded with names that are created by appending
26# the checksum to the actual name, so as not to overwrite any existing files at
27# this stage. If all files are downloaded, the downloaded files are renamed to
28# their real names. On Windows, the old file is removed first to avoid trouble
29# with 'busy' files. If removing a file fails, the file will be moved to a
30# temporary directory, that will be removed when the program starts the next
31# time.
32
33#------------------------------------------------------------------------------
34
35manifestName = "MLXMANIFEST"
36toremoveName = "toremove"
37
38#------------------------------------------------------------------------------
39
40class Manifest(object):
41 """The manifest of the files.
42
43 The manifest file consists of one line for each file. Each line contains 3
44 items separated by tabs:
45 - the path of the file relative to the distribution's root
46 - the size of the file
47 - the MD5 sum of the file."""
48 def __init__(self):
49 """Construct the manifest."""
50 self._files = {}
51
52 @property
53 def files(self):
54 """Get an iterator over the files.
55
56 Each file is returned as a 3-tuple with items as in the file."""
57 for (path, (size, sum)) in self._files.iteritems():
58 yield (path, size, sum)
59
60 def copy(self):
61 """Create a copy of the manifest."""
62 manifest = Manifest()
63 manifest._files = self._files.copy()
64 return manifest
65
66 def addFile(self, path, size, sum):
67 """Add a file to the manifest."""
68 self._files[path] = (size, sum)
69
70 def addFiles(self, baseDirectory, subdirectory):
71 """Add the files in the given directory and subdirectories of it to the
72 manifest."""
73 directory = baseDirectory
74 for d in subdirectory: directory = os.path.join(directory, d)
75
76 for entry in os.listdir(directory):
77 fullPath = os.path.join(directory, entry)
78 if os.path.isfile(fullPath):
79 size = os.stat(fullPath).st_size
80 sum = hashlib.md5()
81 with open(fullPath, "rb") as f:
82 while True:
83 data = f.read(4096)
84 if data: sum.update(data)
85 else: break
86 self.addFile("/".join(subdirectory + [entry]), size,
87 sum.hexdigest())
88 elif os.path.isdir(fullPath):
89 self.addFiles(baseDirectory, subdirectory + [entry])
90
91 def readFrom(self, file):
92 """Read a manifest from the given file object."""
93 for line in iter(file.readline, ""):
94 (path, size, sum) = line.strip().split("\t")
95 self._files[path] = (int(size), sum)
96
97 def writeInto(self, file):
98 """Write the manifest into the file at the given path."""
99 for (path, (size, sum)) in self._files.iteritems():
100 file.write("%s\t%d\t%s\n" % (path, size, sum))
101
102 def compare(self, other):
103 """Compare this manifest with the other one.
104
105 This returns a tuple of two lists:
106 - the files that are either different or are present in other, but not
107 here. Each file is returned as a 3-tuple as with other functions, the
108 size and sum are those of the new file.
109 - the paths of the files that are present here, but not in other."""
110 modifiedAndNew = []
111 for (path, otherSize, otherSum) in other.files:
112 if path in self._files:
113 (size, sum) = self._files[path]
114 if size!=otherSize or sum!=otherSum:
115 modifiedAndNew.append((path, otherSize, otherSum))
116 else:
117 modifiedAndNew.append((path, otherSize, otherSum))
118
119 removed = [path for path in self._files if path not in other._files]
120
121 return (modifiedAndNew, removed)
122
123 def __contains__(self, path):
124 """Determine if the given path is in the manifest."""
125 return path in self._files
126
127 def __getitem__(self, path):
128 """Get data of the file with the given path."""
129 return self._files[path] if path in self._files else None
130
131#------------------------------------------------------------------------------
132
133class ClientListener(object):
134 """A listener that sends any requests via a socket."""
135 def __init__(self, sock):
136 """Construct the listener."""
137 self._sock = sock
138
139 def downloadingManifest(self):
140 """Called when the downloading of the manifest has started."""
141 self._send(["downloadingManifest"])
142
143 def downloadedManifest(self):
144 """Called when the manifest has been downloaded."""
145 self._send(["downloadedManifest"])
146
147 def needSudo(self):
148 """Called when an admin-level program must be called to complete the
149 update.
150
151 This is not valid from a client, as that client is supposed to be the
152 admin-level program :)"""
153 assert False
154
155 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
156 numToRemoveLocal):
157 """Called when starting the downloading of the files."""
158 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
159 str(numToRemove), str(numToRemoveLocal)])
160
161 def setDownloaded(self, downloaded):
162 """Called periodically after downloading a certain amount of data."""
163 self._send(["setDownloaded", str(downloaded)])
164
165 def startRenaming(self):
166 """Called when the renaming of the downloaded files is started."""
167 self._send(["startRenaming"])
168
169 def renamed(self, path, count):
170 """Called when a file has been renamed."""
171 self._send(["renamed", path, str(count)])
172
173 def startRemoving(self):
174 """Called when the removing of files has started."""
175 self._send(["startRemoving"])
176
177 def removed(self, path, count):
178 """Called when a file has been removed."""
179 self._send(["removed", path, str(count)])
180
181 def writingManifest(self):
182 """Called when we have started writing the manifest."""
183 self._send(["writingManifest"])
184
185 def done(self):
186 """Called when the update has completed."""
187 self._send(["done"])
188
189 def failed(self, what):
190 """Called when something has failed."""
191 self._send(["failed", what])
192
193 def _send(self, words):
194 """Send the given words via the socket."""
195 self._sock.send("\t".join(words) + "\n")
196
197#------------------------------------------------------------------------------
198
199def readLocalManifest(directory):
200 """Read the local manifest from the given directory."""
201 manifestPath = os.path.join(directory, manifestName)
202
203 manifest = Manifest()
204 try:
205 with open(manifestPath, "rt") as f:
206 manifest.readFrom(f)
207 except Exception, e:
208 print "Error reading the manifest, ignoring:", str(e)
209 manifest = Manifest()
210
211 return manifest
212
213#------------------------------------------------------------------------------
214
215def prepareUpdate(directory, updateURL, listener):
216 """Prepare the update by downloading the manifest and comparing it with the
217 local one."""
218 manifest = readLocalManifest(directory)
219
220 updateURL += "/" + os.name
221
222 listener.downloadingManifest()
223 f = None
224 try:
225 updateManifest = Manifest()
226 f= urllib2.urlopen(updateURL + "/" + manifestName)
227 updateManifest.readFrom(f)
228
229 except Exception, e:
230 print >> sys.stderr, "Error downloading manifest:", str(e)
231 listener.failed(str(e))
232 return None
233 finally:
234 if f is not None: f.close()
235
236 listener.downloadedManifest()
237
238 (modifiedAndNew, removed) = manifest.compare(updateManifest)
239
240 return (manifest, updateManifest, modifiedAndNew, removed)
241
242#------------------------------------------------------------------------------
243
244def getToremoveFiles(directory):
245 """Add the files to remove from the toremove directory."""
246 toremove = []
247 toremoveDirectory = os.path.join(directory, toremoveName)
248 if os.path.isdir(toremoveDirectory):
249 for entry in os.listdir(toremoveDirectory):
250 toremove.append(os.path.join(toremoveName, entry))
251 return toremove
252
253#------------------------------------------------------------------------------
254
255def createLocalPath(directory, path):
256 """Create a local path from the given manifest path."""
257 localPath = directory
258 for element in path.split("/"):
259 localPath = os.path.join(localPath, element)
260 return localPath
261
262#------------------------------------------------------------------------------
263
264def getToremoveDir(toremoveDir, directory):
265 """Get the path of the directory that will contain the files that are to be
266 removed."""
267 if toremoveDir is None:
268 toremoveDir = os.path.join(directory, toremoveName)
269 try:
270 os.mkdir(toremoveDir)
271 except:
272 pass
273
274 return toremoveDir
275
276#------------------------------------------------------------------------------
277
278def removeFile(toremoveDir, directory, path):
279 """Remove the file at the given path or store it in a temporary directory.
280
281 If the removal of the file fails, it will be stored in a temporary
282 directory. This is useful for files thay may be open and cannot be removed
283 right away."""
284 try:
285 os.remove(path)
286 except:
287 try:
288 sum = hashlib.md5()
289 sum.update(path)
290 toremoveDir = getToremoveDir(toremoveDir, directory)
291 os.rename(path, os.path.join(toremoveDir, sum.hexdigest()))
292 except Exception, e:
293 print "Cannot remove file " + path + ": " + str(e)
294
295#------------------------------------------------------------------------------
296
297def updateFiles(directory, updateURL, listener,
298 manifest, modifiedAndNew, removed, localRemoved):
299 """Update the files according to the given information."""
300 totalSize = 0
301 for (path, size, sum) in modifiedAndNew:
302 totalSize += size
303
304 listener.setTotalSize(len(modifiedAndNew), totalSize,
305 len(removed), len(localRemoved))
306
307 downloaded = 0
308 fin = None
309 toremoveDir = None
310
311 try:
312 updateURL += "/" + os.name
313
314 for (path, size, sum) in modifiedAndNew:
315 targetFile = createLocalPath(directory, path)
316 targetFile += "."
317 targetFile += sum
318
319 targetDirectory = os.path.dirname(targetFile)
320 if not os.path.isdir(targetDirectory):
321 os.makedirs(targetDirectory)
322
323 with open(targetFile, "wb") as fout:
324 fin = urllib2.urlopen(updateURL + "/" + path)
325 while True:
326 data = fin.read(4096)
327 if not data:
328 break
329 fout.write(data)
330 downloaded += len(data)
331 listener.setDownloaded(downloaded)
332 fin.close()
333 fin = None
334
335 listener.startRenaming()
336 count = 0
337 for (path, size, sum) in modifiedAndNew:
338 targetFile = createLocalPath(directory, path)
339
340 downloadedFile = targetFile + "." + sum
341 if os.name=="nt" and os.path.isfile(targetFile):
342 removeFile(toremoveDir, directory, targetFile)
343 os.rename(downloadedFile, targetFile)
344 count += 1
345 listener.renamed(path, count)
346
347 listener.startRemoving()
348 count = 0
349 removed += localRemoved
350 removed.sort(reverse = True)
351 for path in removed:
352 removePath = createLocalPath(directory, path)
353 removeFile(toremoveDir, directory, removePath)
354
355 removeDirectory = os.path.dirname(removePath)
356 try:
357 os.removedirs(removeDirectory)
358 except:
359 pass
360
361 count += 1
362 listener.removed(path, count)
363
364 listener.writingManifest()
365
366 manifestPath = os.path.join(directory, manifestName)
367 with open(manifestPath, "wt") as f:
368 manifest.writeInto(f)
369
370 listener.done()
371 except Exception, e:
372 print >> sys.stderr, "Error:", str(e)
373 listener.failed(str(e))
374
375#------------------------------------------------------------------------------
376
377def isDirectoryWritable(directory):
378 """Determine if the given directory can be written."""
379 checkFile = os.path.join(directory, "writable.chk")
380 try:
381 f = open(checkFile, "wt")
382 f.close()
383 return True
384 except Exception, e:
385 return False
386 finally:
387 try:
388 os.remove(checkFile)
389 except:
390 pass
391
392#------------------------------------------------------------------------------
393
394def processMLXUpdate(buffer, listener):
395 """Process the given buffer supposedly containing a list of commands."""
396 endsWithLine = buffer[-1]=="\n"
397 lines = buffer.splitlines()
398
399 if endsWithLine:
400 buffer = ""
401 else:
402 buffer = lines[-1]
403 lines = lines[:-1]
404
405 for line in lines:
406 words = line.split("\t")
407 try:
408 command = words[0]
409 if command=="downloadingManifest":
410 listener.downloadingManifest()
411 elif command=="downloadedManifest":
412 listener.downloadedManifest()
413 elif command=="setTotalSize":
414 listener.setTotalSize(int(words[1]), long(words[2]),
415 int(words[3]), int(words[4]))
416 elif command=="setDownloaded":
417 listener.setDownloaded(long(words[1]))
418 elif command=="startRenaming":
419 listener.startRenaming()
420 elif command=="renamed":
421 listener.renamed(words[1], int(words[2]))
422 elif command=="startRemoving":
423 listener.startRemoving()
424 elif command=="removed":
425 listener.removed(words[1], int(words[2]))
426 elif command=="writingManifest":
427 listener.writingManifest()
428 elif command=="done":
429 listener.done()
430 elif command=="failed":
431 listener.failed(words[1])
432 except Exception, e:
433 print >> sys.stderr, "Failed to parse line '%s': %s" % \
434 (line, str(e))
435
436 return buffer
437
438#------------------------------------------------------------------------------
439
440def sudoUpdate(directory, updateURL, listener, manifest):
441 """Perform the update via the mlxupdate program."""
442 manifestFD = None
443 manifestFile = None
444 serverSocket = None
445 mlxUpdateSocket = None
446 try:
447 (manifestFD, manifestFile) = tempfile.mkstemp()
448 f = os.fdopen(manifestFD, "wt")
449 try:
450 manifest.writeInto(f)
451 finally:
452 f.close()
453 manifestFD = None
454
455 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
456 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
457 serverSocket.bind(("127.0.0.1", 0))
458 (_host, port) = serverSocket.getsockname()
459 serverSocket.listen(1)
460
461
462 if os.name=="nt":
463 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
464 str(port) + " " + manifestFile, "", 1)
465 else:
466 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
467 str(port), manifestFile],
468 shell = os.name=="nt")
469
470 (mlxUpdateSocket, _) = serverSocket.accept()
471 serverSocket.close()
472 serverSocket = None
473
474 buffer = ""
475 while True:
476 data = mlxUpdateSocket.recv(4096)
477 if not data:
478 break;
479
480 buffer += data
481 buffer = processMLXUpdate(buffer, listener)
482
483 mlxUpdateSocket.close()
484 mlxUpdateSocket = None
485
486 if os.name!="nt":
487 process.wait()
488
489 except Exception, e:
490 print >> sys.stderr, "Failed updating:", str(e)
491 listener.failed(str(e))
492 finally:
493 if serverSocket is not None:
494 try:
495 serverSocket.close()
496 except:
497 pass
498 if mlxUpdateSocket is not None:
499 try:
500 mlxUpdateSocket.close()
501 except:
502 pass
503 if manifestFD is not None:
504 try:
505 os.close(manifestFD)
506 except:
507 pass
508 if manifestFile is not None:
509 try:
510 os.remove(manifestFile)
511 except:
512 pass
513
514
515#------------------------------------------------------------------------------
516
517def update(directory, updateURL, listener, fromGUI = False):
518 """Perform the update."""
519 result = prepareUpdate(directory, updateURL, listener)
520 if result is None:
521 return
522
523 (manifest, updateManifest, modifiedAndNew, removed) = result
524 localRemoved = getToremoveFiles(directory)
525
526 if not modifiedAndNew and not removed and not localRemoved:
527 listener.done()
528 return
529
530 if fromGUI and not isDirectoryWritable(directory):
531 if listener.needSudo():
532 sudoUpdate(directory, updateURL, listener, updateManifest)
533 else:
534 updateFiles(directory, updateURL, listener, updateManifest,
535 modifiedAndNew, removed, localRemoved)
536
537#------------------------------------------------------------------------------
538
539def updateProcess():
540 """This is called in the child process, when we need a child process."""
541 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
542 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
543
544 config = Config()
545 config.load()
546
547 directory = os.path.dirname(sys.argv[0])
548
549 manifest = readLocalManifest(directory)
550
551 updateManifest = Manifest()
552 with open(sys.argv[2], "rt") as f:
553 updateManifest.readFrom(f)
554
555 (modifiedAndNew, removed) = manifest.compare(updateManifest)
556 localRemoved = getToremoveFiles(directory)
557
558 updateFiles(directory, config.updateURL,
559 ClientListener(clientSocket),
560 updateManifest, modifiedAndNew, removed, localRemoved)
561
562#------------------------------------------------------------------------------
563
564def buildManifest(directory):
565 """Build a manifest from the contents of the given directory, into the
566 given directory."""
567
568 manifestPath = os.path.join(directory, manifestName)
569 try:
570 os.remove(manifestPath)
571 except:
572 pass
573
574 manifest = Manifest()
575 manifest.addFiles(directory, [])
576 with open(manifestPath, "wt") as f:
577 manifest.writeInto(f)
578
579#------------------------------------------------------------------------------
580
581# if __name__ == "__main__":
582# manifest1 = Manifest()
583# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
584# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
585# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
586
587# with open("manifest1", "wt") as f:
588# manifest1.writeInto(f)
589
590# manifest2 = Manifest()
591# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
592# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
593# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
594
595# with open("manifest2", "wt") as f:
596# manifest2.writeInto(f)
597
598# manifest1 = Manifest()
599# with open("manifest1", "rt") as f:
600# manifest1.readFrom(f)
601
602# manifest2 = Manifest()
603# with open("manifest2", "rt") as f:
604# manifest2.readFrom(f)
605
606# (modifiedAndNew, removed) = manifest1.compare(manifest2)
607
608# for (path, size, sum) in modifiedAndNew:
609# print "modified or new:", path, size, sum
610
611# for path in removed:
612# print "removed:", path
613
614#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.