source: src/mlx/update.py@ 38:08f7e6592452

Last change on this file since 38:08f7e6592452 was 38:08f7e6592452, checked in by István Váradi <ivaradi@…>, 12 years ago

The status icon is now hidden properly when the program quits, and made restarting nicer
File size: 18.8 KB
Line 
1# Module handling the download of updates
2
3#------------------------------------------------------------------------------
4
5from config import Config
6
7import os
8import sys
9import urllib2
10import tempfile
11import socket
12import subprocess
13import hashlib
14
15#------------------------------------------------------------------------------
16
17manifestName = "MLXMANIFEST"
18toremoveName = "toremove"
19
20#------------------------------------------------------------------------------
21
22class Manifest(object):
23 """The manifest of the files.
24
25 The manifest file consists of one line for each file. Each line contains 3
26 items separated by tabs:
27 - the path of the file relative to the distribution's root
28 - the size of the file
29 - the MD5 sum of the file."""
30 def __init__(self):
31 """Construct the manifest."""
32 self._files = {}
33
34 @property
35 def files(self):
36 """Get an iterator over the files.
37
38 Each file is returned as a 3-tuple with items as in the file."""
39 for (path, (size, sum)) in self._files.iteritems():
40 yield (path, size, sum)
41
42 def addFile(self, path, size, sum):
43 """Add a file to the manifest."""
44 self._files[path] = (size, sum)
45
46 def addFiles(self, baseDirectory, subdirectory):
47 """Add the files in the given directory and subdirectories of it to the
48 manifest."""
49 directory = baseDirectory
50 for d in subdirectory: directory = os.path.join(directory, d)
51
52 for entry in os.listdir(directory):
53 fullPath = os.path.join(directory, entry)
54 if os.path.isfile(fullPath):
55 size = os.stat(fullPath).st_size
56 sum = hashlib.md5()
57 with open(fullPath, "rb") as f:
58 while True:
59 data = f.read(4096)
60 if data: sum.update(data)
61 else: break
62 self.addFile("/".join(subdirectory + [entry]), size,
63 sum.hexdigest())
64 elif os.path.isdir(fullPath):
65 self.addFiles(baseDirectory, subdirectory + [entry])
66
67 def readFrom(self, file):
68 """Read a manifest from the given file object."""
69 for line in iter(file.readline, ""):
70 (path, size, sum) = line.strip().split("\t")
71 self._files[path] = (int(size), sum)
72
73 def writeInto(self, file):
74 """Write the manifest into the file at the given path."""
75 for (path, (size, sum)) in self._files.iteritems():
76 file.write("%s\t%d\t%s\n" % (path, size, sum))
77
78 def compare(self, other):
79 """Compare this manifest with the other one.
80
81 This returns a tuple of two lists:
82 - the files that are either different or are present in other, but not
83 here. Each file is returned as a 3-tuple as with other functions, the
84 size and sum are those of the new file.
85 - the paths of the files that are present here, but not in other."""
86 modifiedAndNew = []
87 for (path, otherSize, otherSum) in other.files:
88 if path in self._files:
89 (size, sum) = self._files[path]
90 if size!=otherSize or sum!=otherSum:
91 modifiedAndNew.append((path, otherSize, otherSum))
92 else:
93 modifiedAndNew.append((path, otherSize, otherSum))
94
95 removed = [path for path in self._files if path not in other._files]
96
97 return (modifiedAndNew, removed)
98
99#------------------------------------------------------------------------------
100
101class ClientListener(object):
102 """A listener that sends any requests via a socket."""
103 def __init__(self, sock):
104 """Construct the listener."""
105 self._sock = sock
106
107 def downloadingManifest(self):
108 """Called when the downloading of the manifest has started."""
109 self._send(["downloadingManifest"])
110
111 def downloadedManifest(self):
112 """Called when the manifest has been downloaded."""
113 self._send(["downloadedManifest"])
114
115 def needSudo(self):
116 """Called when an admin-level program must be called to complete the
117 update.
118
119 This is not valid from a client, as that client is supposed to be the
120 admin-level program :)"""
121 assert False
122
123 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
124 numToRemoveLocal):
125 """Called when starting the downloading of the files."""
126 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
127 str(numToRemove), str(numToRemoveLocal)])
128
129 def setDownloaded(self, downloaded):
130 """Called periodically after downloading a certain amount of data."""
131 self._send(["setDownloaded", str(downloaded)])
132
133 def startRenaming(self):
134 """Called when the renaming of the downloaded files is started."""
135 self._send(["startRenaming"])
136
137 def renamed(self, path, count):
138 """Called when a file has been renamed."""
139 self._send(["renamed", path, str(count)])
140
141 def startRemoving(self):
142 """Called when the removing of files has started."""
143 self._send(["startRemoving"])
144
145 def removed(self, path, count):
146 """Called when a file has been removed."""
147 self._send(["removed", path, str(count)])
148
149 def writingManifest(self):
150 """Called when we have started writing the manifest."""
151 self._send(["writingManifest"])
152
153 def done(self):
154 """Called when the update has completed."""
155 self._send(["done"])
156
157 def failed(self, what):
158 """Called when something has failed."""
159 self._send(["failed", what])
160
161 def _send(self, words):
162 """Send the given words via the socket."""
163 self._sock.send("\t".join(words) + "\n")
164
165#------------------------------------------------------------------------------
166
167def readLocalManifest(directory):
168 """Read the local manifest from the given directory."""
169 manifestPath = os.path.join(directory, manifestName)
170
171 manifest = Manifest()
172 try:
173 with open(manifestPath, "rt") as f:
174 manifest.readFrom(f)
175 except Exception, e:
176 print "Error reading the manifest, ignoring:", str(e)
177 manifest = Manifest()
178
179 return manifest
180
181#------------------------------------------------------------------------------
182
183def prepareUpdate(directory, updateURL, listener):
184 """Prepare the update by downloading the manifest and comparing it with the
185 local one."""
186 manifest = readLocalManifest(directory)
187
188 updateURL += "/" + os.name
189
190 listener.downloadingManifest()
191 f = None
192 try:
193 updateManifest = Manifest()
194 f= urllib2.urlopen(updateURL + "/" + manifestName)
195 updateManifest.readFrom(f)
196
197 except Exception, e:
198 print >> sys.stderr, "Error downloading manifest:", str(e)
199 listener.failed(str(e))
200 return None
201 finally:
202 if f is not None: f.close()
203
204 listener.downloadedManifest()
205
206 (modifiedAndNew, removed) = manifest.compare(updateManifest)
207
208 return (manifest, updateManifest, modifiedAndNew, removed)
209
210#------------------------------------------------------------------------------
211
212def getToremoveFiles(directory):
213 """Add the files to remove from the toremove directory."""
214 toremove = []
215 toremoveDirectory = os.path.join(directory, toremoveName)
216 if os.path.isdir(toremoveDirectory):
217 for entry in os.listdir(toremoveDirectory):
218 toremove.append(os.path.join(toremoveName, entry))
219 return toremove
220
221#------------------------------------------------------------------------------
222
223def createLocalPath(directory, path):
224 """Create a local path from the given manifest path."""
225 localPath = directory
226 for element in path.split("/"):
227 localPath = os.path.join(localPath, element)
228 return localPath
229
230#------------------------------------------------------------------------------
231
232def getToremoveDir(toremoveDir, directory):
233 """Get the path of the directory that will contain the files that are to be
234 removed."""
235 if toremoveDir is None:
236 toremoveDir = os.path.join(directory, toremoveName)
237 try:
238 os.mkdir(toremoveDir)
239 except:
240 pass
241
242 return toremoveDir
243
244#------------------------------------------------------------------------------
245
246def removeFile(toremoveDir, directory, path):
247 """Remove the file at the given path or store it in a temporary directory.
248
249 If the removal of the file fails, it will be stored in a temporary
250 directory. This is useful for files thay may be open and cannot be removed
251 right away."""
252 try:
253 os.remove(path)
254 except:
255 try:
256 sum = hashlib.md5()
257 sum.update(path)
258 toremoveDir = getToremoveDir(toremoveDir, directory)
259 os.rename(path, os.path.join(toremoveDir, sum.hexdigest()))
260 except Exception, e:
261 print "Cannot remove file " + path + ": " + str(e)
262
263#------------------------------------------------------------------------------
264
265def updateFiles(directory, updateURL, listener,
266 manifest, modifiedAndNew, removed, localRemoved):
267 """Update the files according to the given information."""
268 totalSize = 0
269 for (path, size, sum) in modifiedAndNew:
270 totalSize += size
271
272 listener.setTotalSize(len(modifiedAndNew), totalSize,
273 len(removed), len(localRemoved))
274
275 downloaded = 0
276 fin = None
277 toremoveDir = None
278
279 try:
280 updateURL += "/" + os.name
281
282 for (path, size, sum) in modifiedAndNew:
283 targetFile = createLocalPath(directory, path)
284 targetFile += "."
285 targetFile += sum
286
287 targetDirectory = os.path.dirname(targetFile)
288 if not os.path.isdir(targetDirectory):
289 os.makedirs(targetDirectory)
290
291 with open(targetFile, "wb") as fout:
292 fin = urllib2.urlopen(updateURL + "/" + path)
293 while True:
294 data = fin.read(4096)
295 if not data:
296 break
297 fout.write(data)
298 downloaded += len(data)
299 listener.setDownloaded(downloaded)
300 fin.close()
301 fin = None
302
303 listener.startRenaming()
304 count = 0
305 for (path, size, sum) in modifiedAndNew:
306 targetFile = createLocalPath(directory, path)
307
308 downloadedFile = targetFile + "." + sum
309 if os.name=="nt" and os.path.isfile(targetFile):
310 removeFile(toremoveDir, directory, targetFile)
311 os.rename(downloadedFile, targetFile)
312 count += 1
313 listener.renamed(path, count)
314
315 listener.startRemoving()
316 count = 0
317 removed += localRemoved
318 removed.sort(reverse = True)
319 for path in removed:
320 removePath = createLocalPath(directory, path)
321 removeFile(toremoveDir, directory, removePath)
322
323 removeDirectory = os.path.dirname(removePath)
324 try:
325 os.removedirs(removeDirectory)
326 except:
327 pass
328
329 count += 1
330 listener.removed(path, count)
331
332 listener.writingManifest()
333
334 manifestPath = os.path.join(directory, manifestName)
335 with open(manifestPath, "wt") as f:
336 manifest.writeInto(f)
337
338 listener.done()
339 except Exception, e:
340 print >> sys.stderr, "Error:", str(e)
341 listener.failed(str(e))
342
343#------------------------------------------------------------------------------
344
345def isDirectoryWritable(directory):
346 """Determine if the given directory can be written."""
347 checkFile = os.path.join(directory, "writable.chk")
348 try:
349 f = open(checkFile, "wt")
350 f.close()
351 return True
352 except Exception, e:
353 return False
354 finally:
355 try:
356 os.remove(checkFile)
357 except:
358 pass
359
360#------------------------------------------------------------------------------
361
362def processMLXUpdate(buffer, listener):
363 """Process the given buffer supposedly containing a list of commands."""
364 endsWithLine = buffer[-1]=="\n"
365 lines = buffer.splitlines()
366
367 if endsWithLine:
368 buffer = ""
369 else:
370 buffer = lines[-1]
371 lines = lines[:-1]
372
373 for line in lines:
374 words = line.split("\t")
375 try:
376 command = words[0]
377 if command=="downloadingManifest":
378 listener.downloadingManifest()
379 elif command=="downloadedManifest":
380 listener.downloadedManifest()
381 elif command=="setTotalSize":
382 listener.setTotalSize(int(words[1]), long(words[2]),
383 int(words[3]), int(words[4]))
384 elif command=="setDownloaded":
385 listener.setDownloaded(long(words[1]))
386 elif command=="startRenaming":
387 listener.startRenaming()
388 elif command=="renamed":
389 listener.renamed(words[1], int(words[2]))
390 elif command=="startRemoving":
391 listener.startRemoving()
392 elif command=="removed":
393 listener.removed(words[1], int(words[2]))
394 elif command=="writingManifest":
395 listener.writingManifest()
396 elif command=="done":
397 listener.done()
398 elif command=="failed":
399 listener.failed(words[1])
400 except Exception, e:
401 print >> sys.stderr, "Failed to parse line '%s': %s" % \
402 (line, str(e))
403
404 return buffer
405
406#------------------------------------------------------------------------------
407
408def sudoUpdate(directory, updateURL, listener, manifest):
409 """Perform the update via the mlxupdate program."""
410 manifestFD = None
411 manifestFile = None
412 serverSocket = None
413 mlxUpdateSocket = None
414 try:
415 (manifestFD, manifestFile) = tempfile.mkstemp()
416 f = os.fdopen(manifestFD, "wt")
417 try:
418 manifest.writeInto(f)
419 finally:
420 f.close()
421 manifestFD = None
422
423 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
424 serverSocket.bind(("127.0.0.1", 0))
425 (_host, port) = serverSocket.getsockname()
426 serverSocket.listen(1)
427
428 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
429 str(port), manifestFile],
430 shell = os.name=="nt")
431
432 (mlxUpdateSocket, _) = serverSocket.accept()
433 serverSocket.close()
434 serverSocket = None
435
436 buffer = ""
437 while True:
438 data = mlxUpdateSocket.recv(4096)
439 if not data:
440 break;
441
442 buffer += data
443 buffer = processMLXUpdate(buffer, listener)
444
445 mlxUpdateSocket.close()
446 mlxUpdateSocket = None
447
448 process.wait()
449 except Exception, e:
450 print >> sys.stderr, "Failed updating:", str(e)
451 listener.failed(str(e))
452 finally:
453 if serverSocket is not None:
454 try:
455 serverSocket.close()
456 except:
457 pass
458 if mlxUpdateSocket is not None:
459 try:
460 mlxUpdateSocket.close()
461 except:
462 pass
463 if manifestFD is not None:
464 try:
465 os.close(manifestFD)
466 except:
467 pass
468 if manifestFile is not None:
469 try:
470 os.remove(manifestFile)
471 except:
472 pass
473
474
475#------------------------------------------------------------------------------
476
477def update(directory, updateURL, listener, fromGUI = False):
478 """Perform the update."""
479 result = prepareUpdate(directory, updateURL, listener)
480 if result is None:
481 return
482
483 (manifest, updateManifest, modifiedAndNew, removed) = result
484 localRemoved = getToremoveFiles(directory)
485
486 if not modifiedAndNew and not removed and not localRemoved:
487 listener.done()
488 return
489
490 if fromGUI and not isDirectoryWritable(directory):
491 if listener.needSudo():
492 sudoUpdate(directory, updateURL, listener,
493 updateManifest)
494 else:
495 updateFiles(directory, updateURL, listener, updateManifest,
496 modifiedAndNew, removed, localRemoved)
497
498#------------------------------------------------------------------------------
499
500def updateProcess():
501 """This is called in the child process, when we need a child process."""
502 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
503 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
504
505 config = Config()
506
507 directory = os.path.dirname(sys.argv[0])
508
509 manifest = readLocalManifest(directory)
510
511 updateManifest = Manifest()
512 with open(sys.argv[2], "rt") as f:
513 updateManifest.readFrom(f)
514
515 (modifiedAndNew, removed) = manifest.compare(updateManifest)
516 localRemoved = getToremoveFiles(directory)
517
518 updateFiles(directory, config.updateURL,
519 ClientListener(clientSocket),
520 updateManifest, modifiedAndNew, removed, localRemoved)
521
522#------------------------------------------------------------------------------
523
524def buildManifest(directory):
525 """Build a manifest from the contents of the given directory, into the
526 given directory."""
527
528 manifestPath = os.path.join(directory, manifestName)
529 try:
530 os.remove(manifestPath)
531 except:
532 pass
533
534 manifest = Manifest()
535 manifest.addFiles(directory, [])
536 with open(manifestPath, "wt") as f:
537 manifest.writeInto(f)
538
539#------------------------------------------------------------------------------
540
541# if __name__ == "__main__":
542# manifest1 = Manifest()
543# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
544# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
545# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
546
547# with open("manifest1", "wt") as f:
548# manifest1.writeInto(f)
549
550# manifest2 = Manifest()
551# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
552# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
553# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
554
555# with open("manifest2", "wt") as f:
556# manifest2.writeInto(f)
557
558# manifest1 = Manifest()
559# with open("manifest1", "rt") as f:
560# manifest1.readFrom(f)
561
562# manifest2 = Manifest()
563# with open("manifest2", "rt") as f:
564# manifest2.readFrom(f)
565
566# (modifiedAndNew, removed) = manifest1.compare(manifest2)
567
568# for (path, size, sum) in modifiedAndNew:
569# print "modified or new:", path, size, sum
570
571# for path in removed:
572# print "removed:", path
573
574#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.