source: src/mlx/update.py@ 980:2bd74d3bf4b3

python3
Last change on this file since 980:2bd74d3bf4b3 was 977:68c0592b4915, checked in by István Váradi <ivaradi@…>, 6 years ago

The certificates are set for HTTPS access during update (re #347)
File size: 22.2 KB
Line 
1
2from .config import Config
3from .util import utf2unicode
4
5import os
6import sys
7import urllib.request, urllib.error, urllib.parse
8import tempfile
9import socket
10import subprocess
11import hashlib
12import traceback
13import io
14import certifi
15
16if os.name=="nt":
17 import win32api
18
19#------------------------------------------------------------------------------
20
21## @package mlx.update
22#
23# Automatic update handling.
24#
25# The program can update itself automatically. For this purpose it maintains a
26# manifest of the files installed containing the relative paths, sizes and
27# checksums of each files. When the program starts up, this manifest file is
28# downloaded from the update server and is compared to the local one. Then the
29# updated and new files are downloaded with names that are created by appending
30# the checksum to the actual name, so as not to overwrite any existing files at
31# this stage. If all files are downloaded, the downloaded files are renamed to
32# their real names. On Windows, the old file is removed first to avoid trouble
33# with 'busy' files. If removing a file fails, the file will be moved to a
34# temporary directory, that will be removed when the program starts the next
35# time.
36
37#------------------------------------------------------------------------------
38
39manifestName = "MLXMANIFEST"
40toremoveName = "toremove"
41
42#------------------------------------------------------------------------------
43
44class Manifest(object):
45 """The manifest of the files.
46
47 The manifest file consists of one line for each file. Each line contains 3
48 items separated by tabs:
49 - the path of the file relative to the distribution's root
50 - the size of the file
51 - the MD5 sum of the file."""
52 def __init__(self):
53 """Construct the manifest."""
54 self._files = {}
55
56 @property
57 def files(self):
58 """Get an iterator over the files.
59
60 Each file is returned as a 3-tuple with items as in the file."""
61 for (path, (size, sum)) in self._files.items():
62 yield (path, size, sum)
63
64 def copy(self):
65 """Create a copy of the manifest."""
66 manifest = Manifest()
67 manifest._files = self._files.copy()
68 return manifest
69
70 def addFile(self, path, size, sum):
71 """Add a file to the manifest."""
72 self._files[path] = (size, sum)
73
74 def addFiles(self, baseDirectory, subdirectory):
75 """Add the files in the given directory and subdirectories of it to the
76 manifest."""
77 directory = baseDirectory
78 for d in subdirectory: directory = os.path.join(directory, d)
79
80 for entry in os.listdir(directory):
81 fullPath = os.path.join(directory, entry)
82 if os.path.isfile(fullPath):
83 size = os.stat(fullPath).st_size
84 sum = hashlib.md5()
85 with open(fullPath, "rb") as f:
86 while True:
87 data = f.read(4096)
88 if data: sum.update(data)
89 else: break
90 self.addFile("/".join(subdirectory + [entry]), size,
91 sum.hexdigest())
92 elif os.path.isdir(fullPath):
93 self.addFiles(baseDirectory, subdirectory + [entry])
94
95 def readFrom(self, file):
96 """Read a manifest from the given file object."""
97 for line in iter(file.readline, ""):
98 (path, size, sum) = line.strip().split("\t")
99 self._files[path] = (int(size), sum)
100
101 def writeInto(self, file):
102 """Write the manifest into the file at the given path."""
103 for (path, (size, sum)) in self._files.items():
104 file.write("%s\t%d\t%s\n" % (path, size, sum))
105
106 def compare(self, other):
107 """Compare this manifest with the other one.
108
109 This returns a tuple of two lists:
110 - the files that are either different or are present in other, but not
111 here. Each file is returned as a 3-tuple as with other functions, the
112 size and sum are those of the new file.
113 - the paths of the files that are present here, but not in other."""
114 modifiedAndNew = []
115 for (path, otherSize, otherSum) in other.files:
116 if path in self._files:
117 (size, sum) = self._files[path]
118 if size!=otherSize or sum!=otherSum:
119 modifiedAndNew.append((path, otherSize, otherSum))
120 else:
121 modifiedAndNew.append((path, otherSize, otherSum))
122
123 if os.name=="nt":
124 otherFiles = [path.lower() for path in other._files]
125 else:
126 otherFiles = other._files
127
128 removed = [path for path in self._files if
129 (path.lower() if os.name=="nt" else path) not in otherFiles]
130
131 return (modifiedAndNew, removed)
132
133 def __contains__(self, path):
134 """Determine if the given path is in the manifest."""
135 return path in self._files
136
137 def __getitem__(self, path):
138 """Get data of the file with the given path."""
139 return self._files[path] if path in self._files else None
140
141#------------------------------------------------------------------------------
142
143class ClientListener(object):
144 """A listener that sends any requests via a socket."""
145 def __init__(self, sock):
146 """Construct the listener."""
147 self._sock = sock
148
149 def downloadingManifest(self):
150 """Called when the downloading of the manifest has started."""
151 self._send(["downloadingManifest"])
152
153 def downloadedManifest(self):
154 """Called when the manifest has been downloaded."""
155 self._send(["downloadedManifest"])
156
157 def needSudo(self):
158 """Called when an admin-level program must be called to complete the
159 update.
160
161 This is not valid from a client, as that client is supposed to be the
162 admin-level program :)"""
163 assert False
164
165 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
166 numToRemoveLocal):
167 """Called when starting the downloading of the files."""
168 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
169 str(numToRemove), str(numToRemoveLocal)])
170
171 def setDownloaded(self, downloaded):
172 """Called periodically after downloading a certain amount of data."""
173 self._send(["setDownloaded", str(downloaded)])
174
175 def startRenaming(self):
176 """Called when the renaming of the downloaded files is started."""
177 self._send(["startRenaming"])
178
179 def renamed(self, path, count):
180 """Called when a file has been renamed."""
181 self._send(["renamed", path, str(count)])
182
183 def startRemoving(self):
184 """Called when the removing of files has started."""
185 self._send(["startRemoving"])
186
187 def removed(self, path, count):
188 """Called when a file has been removed."""
189 self._send(["removed", path, str(count)])
190
191 def writingManifest(self):
192 """Called when we have started writing the manifest."""
193 self._send(["writingManifest"])
194
195 def done(self):
196 """Called when the update has completed."""
197 self._send(["done"])
198
199 def failed(self, what):
200 """Called when something has failed."""
201 self._send(["failed", what])
202
203 def _send(self, words):
204 """Send the given words via the socket."""
205 self._sock.send(bytes("\t".join(words) + "\n", "utf-8"))
206
207#------------------------------------------------------------------------------
208
209def readLocalManifest(directory):
210 """Read the local manifest from the given directory."""
211 manifestPath = os.path.join(directory, manifestName)
212
213 manifest = Manifest()
214 try:
215 with open(manifestPath, "rt") as f:
216 manifest.readFrom(f)
217 except Exception as e:
218 print("Error reading the manifest, ignoring:", utf2unicode(str(e)))
219 manifest = Manifest()
220
221 return manifest
222
223#------------------------------------------------------------------------------
224
225def prepareUpdate(directory, updateURL, listener):
226 """Prepare the update by downloading the manifest and comparing it with the
227 local one."""
228 manifest = readLocalManifest(directory)
229
230 updateURL += "/" + os.name
231
232 listener.downloadingManifest()
233 f = None
234 try:
235 updateManifest = Manifest()
236 reply = urllib.request.urlopen(updateURL + "/" + manifestName,
237 cafile = certifi.where())
238 charset = reply.headers.get_content_charset()
239 content = reply.read().decode("utf-8" if charset is None else charset)
240 updateManifest.readFrom(io.StringIO(content))
241
242 except Exception as e:
243 error = utf2unicode(str(e))
244 print("Error downloading manifest:", error, file=sys.stderr)
245 listener.failed(error)
246 return None
247 finally:
248 if f is not None: f.close()
249
250 listener.downloadedManifest()
251
252 (modifiedAndNew, removed) = manifest.compare(updateManifest)
253
254 return (manifest, updateManifest, modifiedAndNew, removed)
255
256#------------------------------------------------------------------------------
257
258def getToremoveFiles(directory):
259 """Add the files to remove from the toremove directory."""
260 toremove = []
261 toremoveDirectory = os.path.join(directory, toremoveName)
262 if os.path.isdir(toremoveDirectory):
263 for entry in os.listdir(toremoveDirectory):
264 toremove.append(os.path.join(toremoveName, entry))
265 return toremove
266
267#------------------------------------------------------------------------------
268
269def createLocalPath(directory, path):
270 """Create a local path from the given manifest path."""
271 localPath = directory
272 for element in path.split("/"):
273 localPath = os.path.join(localPath, element)
274 return localPath
275
276#------------------------------------------------------------------------------
277
278def getToremoveDir(toremoveDir, directory):
279 """Get the path of the directory that will contain the files that are to be
280 removed."""
281 if toremoveDir is None:
282 toremoveDir = os.path.join(directory, toremoveName)
283 try:
284 os.mkdir(toremoveDir)
285 except:
286 pass
287
288 return toremoveDir
289
290#------------------------------------------------------------------------------
291
292def removeFile(toremoveDir, directory, path):
293 """Remove the file at the given path or store it in a temporary directory.
294
295 If the removal of the file fails, it will be stored in a temporary
296 directory. This is useful for files thay may be open and cannot be removed
297 right away."""
298 try:
299 os.remove(path)
300 except:
301 try:
302 sum = hashlib.md5()
303 sum.update(bytes(path, "utf-8"))
304 toremoveDir = getToremoveDir(toremoveDir, directory)
305 targetPath = os.path.join(toremoveDir, sum.hexdigest())
306 try:
307 os.remove(targetPath)
308 except:
309 pass
310 os.rename(path, targetPath)
311 except Exception as e:
312 print("Cannot remove file " + path + ": " + utf2unicode(str(e)))
313
314#------------------------------------------------------------------------------
315
316def removeFiles(directory, listener, removed, count):
317 """Remove the given files."""
318 toremoveDir = None
319
320 listener.startRemoving()
321
322 removed.sort(reverse = True)
323 for path in removed:
324 removePath = createLocalPath(directory, path)
325 removeFile(toremoveDir, directory, removePath)
326
327 removeDirectory = os.path.dirname(removePath)
328 try:
329 os.removedirs(removeDirectory)
330 except:
331 pass
332
333 count += 1
334 listener.removed(path, count)
335
336 return count
337
338#------------------------------------------------------------------------------
339
340def updateFiles(directory, updateURL, listener,
341 manifest, modifiedAndNew, removed, localRemoved):
342 """Update the files according to the given information."""
343 totalSize = 0
344 for (path, size, sum) in modifiedAndNew:
345 totalSize += size
346
347 listener.setTotalSize(len(modifiedAndNew), totalSize,
348 len(removed), len(localRemoved))
349
350 downloaded = 0
351 fin = None
352 toremoveDir = None
353
354 try:
355 updateURL += "/" + os.name
356
357 removeCount = 0
358 if localRemoved:
359 removeCount = removeFiles(directory, listener,
360 localRemoved, removeCount)
361
362 for (path, size, sum) in modifiedAndNew:
363 targetFile = createLocalPath(directory, path)
364 targetFile += "."
365 targetFile += sum
366
367 targetDirectory = os.path.dirname(targetFile)
368 if targetDirectory and not os.path.isdir(targetDirectory):
369 os.makedirs(targetDirectory)
370
371 with open(targetFile, "wb") as fout:
372 fin = urllib.request.urlopen(updateURL + "/" + path,
373 cafile = certifi.where())
374 while True:
375 data = fin.read(256*1024)
376 if not data:
377 break
378 fout.write(data)
379 downloaded += len(data)
380 listener.setDownloaded(downloaded)
381 fin.close()
382 fin = None
383
384 listener.startRenaming()
385 count = 0
386 for (path, size, sum) in modifiedAndNew:
387 targetFile = createLocalPath(directory, path)
388
389 downloadedFile = targetFile + "." + sum
390 if os.name=="nt" and os.path.isfile(targetFile):
391 removeFile(toremoveDir, directory, targetFile)
392 os.rename(downloadedFile, targetFile)
393 count += 1
394 listener.renamed(path, count)
395
396 removeFiles(directory, listener, removed, removeCount)
397
398 listener.writingManifest()
399
400 manifestPath = os.path.join(directory, manifestName)
401 with open(manifestPath, "wt") as f:
402 manifest.writeInto(f)
403
404 listener.done()
405 except Exception as e:
406 exc = traceback.format_exc()
407 print(utf2unicode(exc), file=sys.stderr)
408
409 error = utf2unicode(str(e))
410 print("Error:", error, file=sys.stderr)
411
412 listener.failed(error)
413
414#------------------------------------------------------------------------------
415
416def isDirectoryWritable(directory):
417 """Determine if the given directory can be written."""
418 checkFile = os.path.join(directory, "writable.chk")
419 try:
420 f = open(checkFile, "wt")
421 f.close()
422 return True
423 except Exception as e:
424 return False
425 finally:
426 try:
427 os.remove(checkFile)
428 except:
429 pass
430
431#------------------------------------------------------------------------------
432
433def processMLXUpdate(buffer, listener):
434 """Process the given buffer supposedly containing a list of commands."""
435 endsWithLine = buffer[-1]=="\n"
436 lines = buffer.splitlines()
437
438 if endsWithLine:
439 buffer = ""
440 else:
441 buffer = lines[-1]
442 lines = lines[:-1]
443
444 for line in lines:
445 words = line.split("\t")
446 try:
447 command = words[0]
448 if command=="downloadingManifest":
449 listener.downloadingManifest()
450 elif command=="downloadedManifest":
451 listener.downloadedManifest()
452 elif command=="setTotalSize":
453 listener.setTotalSize(int(words[1]), int(words[2]),
454 int(words[3]), int(words[4]))
455 elif command=="setDownloaded":
456 listener.setDownloaded(int(words[1]))
457 elif command=="startRenaming":
458 listener.startRenaming()
459 elif command=="renamed":
460 listener.renamed(words[1], int(words[2]))
461 elif command=="startRemoving":
462 listener.startRemoving()
463 elif command=="removed":
464 listener.removed(words[1], int(words[2]))
465 elif command=="writingManifest":
466 listener.writingManifest()
467 elif command=="done":
468 listener.done()
469 elif command=="failed":
470 listener.failed(words[1])
471 except Exception as e:
472 print("Failed to parse line '%s': %s" % \
473 (line, utf2unicode(str(e))), file=sys.stderr)
474
475 return buffer
476
477#------------------------------------------------------------------------------
478
479def sudoUpdate(directory, updateURL, listener, manifest):
480 """Perform the update via the mlxupdate program."""
481 manifestFD = None
482 manifestFile = None
483 serverSocket = None
484 mlxUpdateSocket = None
485 try:
486 (manifestFD, manifestFile) = tempfile.mkstemp()
487 f = os.fdopen(manifestFD, "wt")
488 try:
489 manifest.writeInto(f)
490 finally:
491 f.close()
492 manifestFD = None
493
494 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
495 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
496 serverSocket.bind(("127.0.0.1", 0))
497 (_host, port) = serverSocket.getsockname()
498 serverSocket.listen(1)
499
500
501 if os.name=="nt":
502 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
503 str(port) + " " + manifestFile + " " + updateURL, "", 1)
504 else:
505 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
506 str(port), manifestFile, updateURL],
507 shell = os.name=="nt")
508
509 (mlxUpdateSocket, _) = serverSocket.accept()
510 serverSocket.close()
511 serverSocket = None
512
513 buffer = ""
514 while True:
515 data = mlxUpdateSocket.recv(4096)
516 if not data:
517 break;
518
519 buffer += str(data, "utf-8")
520 buffer = processMLXUpdate(buffer, listener)
521
522 mlxUpdateSocket.close()
523 mlxUpdateSocket = None
524
525 if os.name!="nt":
526 process.wait()
527
528 except Exception as e:
529 error = utf2unicode(str(e))
530 print("Failed updating:", error, file=sys.stderr)
531 listener.failed(error)
532 finally:
533 if serverSocket is not None:
534 try:
535 serverSocket.close()
536 except:
537 pass
538 if mlxUpdateSocket is not None:
539 try:
540 mlxUpdateSocket.close()
541 except:
542 pass
543 if manifestFD is not None:
544 try:
545 os.close(manifestFD)
546 except:
547 pass
548 if manifestFile is not None:
549 try:
550 os.remove(manifestFile)
551 except:
552 pass
553
554
555#------------------------------------------------------------------------------
556
557def update(directory, updateURL, listener, fromGUI = False):
558 """Perform the update."""
559 try:
560 result = prepareUpdate(directory, updateURL, listener)
561 if result is None:
562 return
563
564 (manifest, updateManifest, modifiedAndNew, removed) = result
565 localRemoved = getToremoveFiles(directory)
566
567 if not modifiedAndNew and not removed and not localRemoved:
568 listener.done()
569 return
570
571 if fromGUI and not isDirectoryWritable(directory):
572 if listener.needSudo():
573 sudoUpdate(directory, updateURL, listener, updateManifest)
574 else:
575 updateFiles(directory, updateURL, listener, updateManifest,
576 modifiedAndNew, removed, localRemoved)
577 except Exception as e:
578 exc = traceback.format_exc()
579 print(utf2unicode(exc), file=sys.stderr)
580
581 error = utf2unicode(str(e))
582 print("Update error:", error, file=sys.stderr)
583
584 listener.failed(error)
585
586#------------------------------------------------------------------------------
587
588def updateProcess():
589 """This is called in the child process, when we need a child process."""
590 try:
591 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
592 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
593
594 directory = os.path.dirname(sys.argv[0])
595
596 manifest = readLocalManifest(directory)
597
598 updateManifest = Manifest()
599 with open(sys.argv[2], "rt") as f:
600 updateManifest.readFrom(f)
601
602 (modifiedAndNew, removed) = manifest.compare(updateManifest)
603 localRemoved = getToremoveFiles(directory)
604
605 updateFiles(directory, sys.argv[3],
606 ClientListener(clientSocket),
607 updateManifest, modifiedAndNew, removed, localRemoved)
608 except:
609 exc = traceback.format_exc()
610 print(utf2unicode(exc), file=sys.stderr)
611
612#------------------------------------------------------------------------------
613
614def buildManifest(directory):
615 """Build a manifest from the contents of the given directory, into the
616 given directory."""
617
618 manifestPath = os.path.join(directory, manifestName)
619 try:
620 os.remove(manifestPath)
621 except:
622 pass
623
624 manifest = Manifest()
625 manifest.addFiles(directory, [])
626 with open(manifestPath, "wt") as f:
627 manifest.writeInto(f)
628
629#------------------------------------------------------------------------------
630
631# if __name__ == "__main__":
632# manifest1 = Manifest()
633# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
634# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
635# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
636
637# with open("manifest1", "wt") as f:
638# manifest1.writeInto(f)
639
640# manifest2 = Manifest()
641# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
642# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
643# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
644
645# with open("manifest2", "wt") as f:
646# manifest2.writeInto(f)
647
648# manifest1 = Manifest()
649# with open("manifest1", "rt") as f:
650# manifest1.readFrom(f)
651
652# manifest2 = Manifest()
653# with open("manifest2", "rt") as f:
654# manifest2.readFrom(f)
655
656# (modifiedAndNew, removed) = manifest1.compare(manifest2)
657
658# for (path, size, sum) in modifiedAndNew:
659# print "modified or new:", path, size, sum
660
661# for path in removed:
662# print "removed:", path
663
664#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.