source: src/mlx/update.py@ 1191:0db341c8c3b3

python3 tip
Last change on this file since 1191:0db341c8c3b3 was 1191:0db341c8c3b3, checked in by István Váradi <ivaradi@…>, 4 days ago

context is used instead of cafile in urlopen()
File size: 22.3 KB
Line 
1
2from .config import Config
3from .util import utf2unicode
4
5import os
6import sys
7import urllib.request, urllib.error, urllib.parse
8import tempfile
9import socket
10import subprocess
11import hashlib
12import traceback
13import io
14
15if os.name=="nt":
16 import win32api
17
18#------------------------------------------------------------------------------
19
20## @package mlx.update
21#
22# Automatic update handling.
23#
24# The program can update itself automatically. For this purpose it maintains a
25# manifest of the files installed containing the relative paths, sizes and
26# checksums of each files. When the program starts up, this manifest file is
27# downloaded from the update server and is compared to the local one. Then the
28# updated and new files are downloaded with names that are created by appending
29# the checksum to the actual name, so as not to overwrite any existing files at
30# this stage. If all files are downloaded, the downloaded files are renamed to
31# their real names. On Windows, the old file is removed first to avoid trouble
32# with 'busy' files. If removing a file fails, the file will be moved to a
33# temporary directory, that will be removed when the program starts the next
34# time.
35
36#------------------------------------------------------------------------------
37
38manifestName = "MLXMANIFEST"
39toremoveName = "toremove"
40
41#------------------------------------------------------------------------------
42
43class Manifest(object):
44 """The manifest of the files.
45
46 The manifest file consists of one line for each file. Each line contains 3
47 items separated by tabs:
48 - the path of the file relative to the distribution's root
49 - the size of the file
50 - the MD5 sum of the file."""
51 def __init__(self):
52 """Construct the manifest."""
53 self._files = {}
54
55 @property
56 def files(self):
57 """Get an iterator over the files.
58
59 Each file is returned as a 3-tuple with items as in the file."""
60 for (path, (size, sum)) in self._files.items():
61 yield (path, size, sum)
62
63 def copy(self):
64 """Create a copy of the manifest."""
65 manifest = Manifest()
66 manifest._files = self._files.copy()
67 return manifest
68
69 def addFile(self, path, size, sum):
70 """Add a file to the manifest."""
71 self._files[path] = (size, sum)
72
73 def addFiles(self, baseDirectory, subdirectory):
74 """Add the files in the given directory and subdirectories of it to the
75 manifest."""
76 directory = baseDirectory
77 for d in subdirectory: directory = os.path.join(directory, d)
78
79 for entry in os.listdir(directory):
80 fullPath = os.path.join(directory, entry)
81 if os.path.isfile(fullPath):
82 size = os.stat(fullPath).st_size
83 sum = hashlib.md5()
84 with open(fullPath, "rb") as f:
85 while True:
86 data = f.read(4096)
87 if data: sum.update(data)
88 else: break
89 self.addFile("/".join(subdirectory + [entry]), size,
90 sum.hexdigest())
91 elif os.path.isdir(fullPath):
92 self.addFiles(baseDirectory, subdirectory + [entry])
93
94 def readFrom(self, file):
95 """Read a manifest from the given file object."""
96 for line in iter(file.readline, ""):
97 (path, size, sum) = line.strip().split("\t")
98 self._files[path] = (int(size), sum)
99
100 def writeInto(self, file):
101 """Write the manifest into the file at the given path."""
102 for (path, (size, sum)) in self._files.items():
103 file.write("%s\t%d\t%s\n" % (path, size, sum))
104
105 def compare(self, other):
106 """Compare this manifest with the other one.
107
108 This returns a tuple of two lists:
109 - the files that are either different or are present in other, but not
110 here. Each file is returned as a 3-tuple as with other functions, the
111 size and sum are those of the new file.
112 - the paths of the files that are present here, but not in other."""
113 modifiedAndNew = []
114 for (path, otherSize, otherSum) in other.files:
115 if path in self._files:
116 (size, sum) = self._files[path]
117 if size!=otherSize or sum!=otherSum:
118 modifiedAndNew.append((path, otherSize, otherSum))
119 else:
120 modifiedAndNew.append((path, otherSize, otherSum))
121
122 if os.name=="nt":
123 otherFiles = [path.lower() for path in other._files]
124 else:
125 otherFiles = other._files
126
127 removed = [path for path in self._files if
128 (path.lower() if os.name=="nt" else path) not in otherFiles]
129
130 return (modifiedAndNew, removed)
131
132 def __contains__(self, path):
133 """Determine if the given path is in the manifest."""
134 return path in self._files
135
136 def __getitem__(self, path):
137 """Get data of the file with the given path."""
138 return self._files[path] if path in self._files else None
139
140#------------------------------------------------------------------------------
141
142class ClientListener(object):
143 """A listener that sends any requests via a socket."""
144 def __init__(self, sock):
145 """Construct the listener."""
146 self._sock = sock
147
148 def downloadingManifest(self):
149 """Called when the downloading of the manifest has started."""
150 self._send(["downloadingManifest"])
151
152 def downloadedManifest(self):
153 """Called when the manifest has been downloaded."""
154 self._send(["downloadedManifest"])
155
156 def needSudo(self):
157 """Called when an admin-level program must be called to complete the
158 update.
159
160 This is not valid from a client, as that client is supposed to be the
161 admin-level program :)"""
162 assert False
163
164 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
165 numToRemoveLocal):
166 """Called when starting the downloading of the files."""
167 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
168 str(numToRemove), str(numToRemoveLocal)])
169
170 def setDownloaded(self, downloaded):
171 """Called periodically after downloading a certain amount of data."""
172 self._send(["setDownloaded", str(downloaded)])
173
174 def startRenaming(self):
175 """Called when the renaming of the downloaded files is started."""
176 self._send(["startRenaming"])
177
178 def renamed(self, path, count):
179 """Called when a file has been renamed."""
180 self._send(["renamed", path, str(count)])
181
182 def startRemoving(self):
183 """Called when the removing of files has started."""
184 self._send(["startRemoving"])
185
186 def removed(self, path, count):
187 """Called when a file has been removed."""
188 self._send(["removed", path, str(count)])
189
190 def writingManifest(self):
191 """Called when we have started writing the manifest."""
192 self._send(["writingManifest"])
193
194 def done(self):
195 """Called when the update has completed."""
196 self._send(["done"])
197
198 def failed(self, what):
199 """Called when something has failed."""
200 self._send(["failed", what])
201
202 def _send(self, words):
203 """Send the given words via the socket."""
204 self._sock.send(bytes("\t".join(words) + "\n", "utf-8"))
205
206#------------------------------------------------------------------------------
207
208def readLocalManifest(directory):
209 """Read the local manifest from the given directory."""
210 manifestPath = os.path.join(directory, manifestName)
211
212 manifest = Manifest()
213 try:
214 with open(manifestPath, "rt") as f:
215 manifest.readFrom(f)
216 except Exception as e:
217 print("Error reading the manifest, ignoring:", utf2unicode(str(e)))
218 manifest = Manifest()
219
220 return manifest
221
222#------------------------------------------------------------------------------
223
224def prepareUpdate(directory, updateURL, listener):
225 """Prepare the update by downloading the manifest and comparing it with the
226 local one."""
227 manifest = readLocalManifest(directory)
228
229 updateURL += "/" + os.name
230
231 listener.downloadingManifest()
232 f = None
233 try:
234 from .common import sslContext
235
236 updateManifest = Manifest()
237 reply = urllib.request.urlopen(updateURL + "/" + manifestName,
238 context = sslContext)
239 charset = reply.headers.get_content_charset()
240 content = reply.read().decode("utf-8" if charset is None else charset)
241 updateManifest.readFrom(io.StringIO(content))
242
243 except Exception as e:
244 error = utf2unicode(str(e))
245 print("Error downloading manifest:", error, file=sys.stderr)
246 listener.failed(error)
247 return None
248 finally:
249 if f is not None: f.close()
250
251 listener.downloadedManifest()
252
253 (modifiedAndNew, removed) = manifest.compare(updateManifest)
254
255 return (manifest, updateManifest, modifiedAndNew, removed)
256
257#------------------------------------------------------------------------------
258
259def getToremoveFiles(directory):
260 """Add the files to remove from the toremove directory."""
261 toremove = []
262 toremoveDirectory = os.path.join(directory, toremoveName)
263 if os.path.isdir(toremoveDirectory):
264 for entry in os.listdir(toremoveDirectory):
265 toremove.append(os.path.join(toremoveName, entry))
266 return toremove
267
268#------------------------------------------------------------------------------
269
270def createLocalPath(directory, path):
271 """Create a local path from the given manifest path."""
272 localPath = directory
273 for element in path.split("/"):
274 localPath = os.path.join(localPath, element)
275 return localPath
276
277#------------------------------------------------------------------------------
278
279def getToremoveDir(toremoveDir, directory):
280 """Get the path of the directory that will contain the files that are to be
281 removed."""
282 if toremoveDir is None:
283 toremoveDir = os.path.join(directory, toremoveName)
284 try:
285 os.mkdir(toremoveDir)
286 except:
287 pass
288
289 return toremoveDir
290
291#------------------------------------------------------------------------------
292
293def removeFile(toremoveDir, directory, path):
294 """Remove the file at the given path or store it in a temporary directory.
295
296 If the removal of the file fails, it will be stored in a temporary
297 directory. This is useful for files thay may be open and cannot be removed
298 right away."""
299 try:
300 os.remove(path)
301 except:
302 try:
303 sum = hashlib.md5()
304 sum.update(bytes(path, "utf-8"))
305 toremoveDir = getToremoveDir(toremoveDir, directory)
306 targetPath = os.path.join(toremoveDir, sum.hexdigest())
307 try:
308 os.remove(targetPath)
309 except:
310 pass
311 os.rename(path, targetPath)
312 except Exception as e:
313 print("Cannot remove file " + path + ": " + utf2unicode(str(e)))
314
315#------------------------------------------------------------------------------
316
317def removeFiles(directory, listener, removed, count):
318 """Remove the given files."""
319 toremoveDir = None
320
321 listener.startRemoving()
322
323 removed.sort(reverse = True)
324 for path in removed:
325 removePath = createLocalPath(directory, path)
326 removeFile(toremoveDir, directory, removePath)
327
328 removeDirectory = os.path.dirname(removePath)
329 try:
330 os.removedirs(removeDirectory)
331 except:
332 pass
333
334 count += 1
335 listener.removed(path, count)
336
337 return count
338
339#------------------------------------------------------------------------------
340
341def updateFiles(directory, updateURL, listener,
342 manifest, modifiedAndNew, removed, localRemoved):
343 """Update the files according to the given information."""
344 totalSize = 0
345 for (path, size, sum) in modifiedAndNew:
346 totalSize += size
347
348 listener.setTotalSize(len(modifiedAndNew), totalSize,
349 len(removed), len(localRemoved))
350
351 downloaded = 0
352 fin = None
353 toremoveDir = None
354
355 try:
356 updateURL += "/" + os.name
357
358 removeCount = 0
359 if localRemoved:
360 removeCount = removeFiles(directory, listener,
361 localRemoved, removeCount)
362
363 for (path, size, sum) in modifiedAndNew:
364 targetFile = createLocalPath(directory, path)
365 targetFile += "."
366 targetFile += sum
367
368 targetDirectory = os.path.dirname(targetFile)
369 if targetDirectory and not os.path.isdir(targetDirectory):
370 os.makedirs(targetDirectory)
371
372 with open(targetFile, "wb") as fout:
373 from .common import sslContext
374
375 fin = urllib.request.urlopen(updateURL + "/" + path,
376 context = sslContext)
377 while True:
378 data = fin.read(256*1024)
379 if not data:
380 break
381 fout.write(data)
382 downloaded += len(data)
383 listener.setDownloaded(downloaded)
384 fin.close()
385 fin = None
386
387 listener.startRenaming()
388 count = 0
389 for (path, size, sum) in modifiedAndNew:
390 targetFile = createLocalPath(directory, path)
391
392 downloadedFile = targetFile + "." + sum
393 if os.name=="nt" and os.path.isfile(targetFile):
394 removeFile(toremoveDir, directory, targetFile)
395 os.rename(downloadedFile, targetFile)
396 count += 1
397 listener.renamed(path, count)
398
399 removeFiles(directory, listener, removed, removeCount)
400
401 listener.writingManifest()
402
403 manifestPath = os.path.join(directory, manifestName)
404 with open(manifestPath, "wt") as f:
405 manifest.writeInto(f)
406
407 listener.done()
408 except Exception as e:
409 exc = traceback.format_exc()
410 print(utf2unicode(exc), file=sys.stderr)
411
412 error = utf2unicode(str(e))
413 print("Error:", error, file=sys.stderr)
414
415 listener.failed(error)
416
417#------------------------------------------------------------------------------
418
419def isDirectoryWritable(directory):
420 """Determine if the given directory can be written."""
421 checkFile = os.path.join(directory, "writable.chk")
422 try:
423 f = open(checkFile, "wt")
424 f.close()
425 return True
426 except Exception as e:
427 return False
428 finally:
429 try:
430 os.remove(checkFile)
431 except:
432 pass
433
434#------------------------------------------------------------------------------
435
436def processMLXUpdate(buffer, listener):
437 """Process the given buffer supposedly containing a list of commands."""
438 endsWithLine = buffer[-1]=="\n"
439 lines = buffer.splitlines()
440
441 if endsWithLine:
442 buffer = ""
443 else:
444 buffer = lines[-1]
445 lines = lines[:-1]
446
447 for line in lines:
448 words = line.split("\t")
449 try:
450 command = words[0]
451 if command=="downloadingManifest":
452 listener.downloadingManifest()
453 elif command=="downloadedManifest":
454 listener.downloadedManifest()
455 elif command=="setTotalSize":
456 listener.setTotalSize(int(words[1]), int(words[2]),
457 int(words[3]), int(words[4]))
458 elif command=="setDownloaded":
459 listener.setDownloaded(int(words[1]))
460 elif command=="startRenaming":
461 listener.startRenaming()
462 elif command=="renamed":
463 listener.renamed(words[1], int(words[2]))
464 elif command=="startRemoving":
465 listener.startRemoving()
466 elif command=="removed":
467 listener.removed(words[1], int(words[2]))
468 elif command=="writingManifest":
469 listener.writingManifest()
470 elif command=="done":
471 listener.done()
472 elif command=="failed":
473 listener.failed(words[1])
474 except Exception as e:
475 print("Failed to parse line '%s': %s" % \
476 (line, utf2unicode(str(e))), file=sys.stderr)
477
478 return buffer
479
480#------------------------------------------------------------------------------
481
482def sudoUpdate(directory, updateURL, listener, manifest):
483 """Perform the update via the mlxupdate program."""
484 manifestFD = None
485 manifestFile = None
486 serverSocket = None
487 mlxUpdateSocket = None
488 try:
489 (manifestFD, manifestFile) = tempfile.mkstemp()
490 f = os.fdopen(manifestFD, "wt")
491 try:
492 manifest.writeInto(f)
493 finally:
494 f.close()
495 manifestFD = None
496
497 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
498 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
499 serverSocket.bind(("127.0.0.1", 0))
500 (_host, port) = serverSocket.getsockname()
501 serverSocket.listen(1)
502
503
504 if os.name=="nt":
505 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
506 str(port) + " " + manifestFile + " " + updateURL, "", 1)
507 else:
508 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
509 str(port), manifestFile, updateURL],
510 shell = os.name=="nt")
511
512 (mlxUpdateSocket, _) = serverSocket.accept()
513 serverSocket.close()
514 serverSocket = None
515
516 buffer = ""
517 while True:
518 data = mlxUpdateSocket.recv(4096)
519 if not data:
520 break;
521
522 buffer += str(data, "utf-8")
523 buffer = processMLXUpdate(buffer, listener)
524
525 mlxUpdateSocket.close()
526 mlxUpdateSocket = None
527
528 if os.name!="nt":
529 process.wait()
530
531 except Exception as e:
532 error = utf2unicode(str(e))
533 print("Failed updating:", error, file=sys.stderr)
534 listener.failed(error)
535 finally:
536 if serverSocket is not None:
537 try:
538 serverSocket.close()
539 except:
540 pass
541 if mlxUpdateSocket is not None:
542 try:
543 mlxUpdateSocket.close()
544 except:
545 pass
546 if manifestFD is not None:
547 try:
548 os.close(manifestFD)
549 except:
550 pass
551 if manifestFile is not None:
552 try:
553 os.remove(manifestFile)
554 except:
555 pass
556
557
558#------------------------------------------------------------------------------
559
560def update(directory, updateURL, listener, fromGUI = False):
561 """Perform the update."""
562 try:
563 result = prepareUpdate(directory, updateURL, listener)
564 if result is None:
565 return
566
567 (manifest, updateManifest, modifiedAndNew, removed) = result
568 localRemoved = getToremoveFiles(directory)
569
570 if not modifiedAndNew and not removed and not localRemoved:
571 listener.done()
572 return
573
574 if fromGUI and not isDirectoryWritable(directory):
575 if listener.needSudo():
576 sudoUpdate(directory, updateURL, listener, updateManifest)
577 else:
578 updateFiles(directory, updateURL, listener, updateManifest,
579 modifiedAndNew, removed, localRemoved)
580 except Exception as e:
581 exc = traceback.format_exc()
582 print(utf2unicode(exc), file=sys.stderr)
583
584 error = utf2unicode(str(e))
585 print("Update error:", error, file=sys.stderr)
586
587 listener.failed(error)
588
589#------------------------------------------------------------------------------
590
591def updateProcess():
592 """This is called in the child process, when we need a child process."""
593 try:
594 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
595 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
596
597 directory = os.path.dirname(sys.argv[0])
598
599 manifest = readLocalManifest(directory)
600
601 updateManifest = Manifest()
602 with open(sys.argv[2], "rt") as f:
603 updateManifest.readFrom(f)
604
605 (modifiedAndNew, removed) = manifest.compare(updateManifest)
606 localRemoved = getToremoveFiles(directory)
607
608 updateFiles(directory, sys.argv[3],
609 ClientListener(clientSocket),
610 updateManifest, modifiedAndNew, removed, localRemoved)
611 except:
612 exc = traceback.format_exc()
613 print(utf2unicode(exc), file=sys.stderr)
614
615#------------------------------------------------------------------------------
616
617def buildManifest(directory):
618 """Build a manifest from the contents of the given directory, into the
619 given directory."""
620
621 manifestPath = os.path.join(directory, manifestName)
622 try:
623 os.remove(manifestPath)
624 except:
625 pass
626
627 manifest = Manifest()
628 manifest.addFiles(directory, [])
629 with open(manifestPath, "wt") as f:
630 manifest.writeInto(f)
631
632#------------------------------------------------------------------------------
633
634# if __name__ == "__main__":
635# manifest1 = Manifest()
636# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
637# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
638# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
639
640# with open("manifest1", "wt") as f:
641# manifest1.writeInto(f)
642
643# manifest2 = Manifest()
644# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
645# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
646# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
647
648# with open("manifest2", "wt") as f:
649# manifest2.writeInto(f)
650
651# manifest1 = Manifest()
652# with open("manifest1", "rt") as f:
653# manifest1.readFrom(f)
654
655# manifest2 = Manifest()
656# with open("manifest2", "rt") as f:
657# manifest2.readFrom(f)
658
659# (modifiedAndNew, removed) = manifest1.compare(manifest2)
660
661# for (path, size, sum) in modifiedAndNew:
662# print "modified or new:", path, size, sum
663
664# for path in removed:
665# print "removed:", path
666
667#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.