sp update (#242)

* Initial commit for "sp update"

* Fixed update detection

Added missing message prefixes

* Changed ' to "

* Use Boost.Filesystem instead if IFileSystem

IFileSystem had some problems with not seeing all files on Linux.

* Added platform specific parts to stage 1

* Updated get_artifacts()

* Added "sp update" command to the wiki

* Updated update instructions on the wiki

* Removed unused macros

* Don't create copies of bfs::path

Author: Ayuto

addons/source-python/docs/source-python/source/general/sp-commands.rst

@@ -493,3 +493,15 @@ Unload a plugin by name.
 
     // Unload the plugin 'test'
     sp plugin unload test
+
+
+update
+------
+
+Update Source.Python to the latest version. A restart of the server is required
+to apply the new update.
+
+.. code-block:: none
+
+    // Usage
+    // sp update

addons/source-python/docs/source-python/source/general/updating.rst

@@ -1,6 +1,17 @@
 Updating
 ========
 
+Manually
+--------
+
 1. Delete the entire ``../addons/source-python/data/source-python/`` directory.
 2. Delete the entire ``../addons/source-python/packages/source-python/`` directory.
-3. Continue with the :doc:`installation tutorial <installation>`.
+3. Delete the entire ``../addons/source-python/docs/source-python/`` directory.
+4. Delete the entire ``../addons/source-python/Python3/`` directory.
+5. Continue with the :doc:`installation tutorial <installation>`.
+
+
+Automatically
+-------------
+
+1. Run the command ``sp update`` and follow the instructions printed to the console (mostly, it's just a restart).

addons/source-python/packages/source-python/core/command/__init__.py

@@ -22,6 +22,11 @@
 from core import core_logger
 from core import create_checksum
 from core import SOURCE_ENGINE_BRANCH
+from core.update import clean_update_dir
+from core.update import download_latest_version
+from core.update import apply_update_stage1
+from core.version import get_last_successful_build_number
+from core.version import is_unversioned
 from core.version import VERSION
 from core.version import GIT_COMMIT
 #   Engines
@@ -158,6 +163,27 @@ def print_info(info):
         f'Checksum      : {checksum}{result}\n{sep}\n')
 
 
+@core_command.server_sub_command(['update'])
+def update_sp(info):
+    """Update Source.Python to the latest version. A restart of the server is
+    required.
+    """
+    if not is_unversioned() and VERSION >= get_last_successful_build_number():
+        core_command_logger.log_message('No new version available.')
+        return
+        
+    # Make sure there is a clean update directory
+    clean_update_dir()
+    try:
+        download_latest_version()
+        apply_update_stage1()
+    except:
+        # Make sure to leave a clean update directory, so the loader doesn't
+        # get confused.
+        clean_update_dir()
+        raise
+    
+
 # =============================================================================
 # >> DESCRIPTION
 # =============================================================================

addons/source-python/packages/source-python/core/update.py

@@ -6,28 +6,46 @@
 # >> IMPORTS
 # =============================================================================
 # Python Imports
+import json
+import time
+
 from zipfile import ZipFile
 from urllib.request import urlopen
 
 # Source.Python Imports
 #   Core
+from core import PLATFORM
 from core import core_logger
+from core import SOURCE_ENGINE_BRANCH
 #   Paths
+from paths import ADDONS_PATH
+from paths import GAME_PATH
+from paths import UPDATE_PATH
 from paths import DATA_PATH
 from paths import SP_DATA_PATH
+#   KeyValues
+from keyvalues import KeyValues
 
 
 # =============================================================================
 # >> ALL DECLARATION
 # =============================================================================
 __all__ = (
+    'ARTIFACTS_URL',
+    'BASE_DOWNLOAD_URL',
     'CHECKSUM_URL',
     'DATA_URL',
     'DATA_ZIP_FILE',
-    'download_latest_data',
-    'get_latest_data_checksum',
-    'is_new_data_available',
-    'unpack_data',
+    'apply_update_stage1'
+    'clean_update_dir'
+    'download_file'
+    'download_latest_data'
+    'download_latest_version'
+    'get_artifacts'
+    'get_download_url'
+    'get_latest_data_checksum'
+    'is_new_data_available'
+    'unpack_data'
     'update_data'
 )
 
@@ -38,9 +56,22 @@
 # Don't use __getattr__ here. 'update' is a method of the _LogInstance class.
 update_logger = core_logger['update']
 
+BINARY_EXT = 'so' if PLATFORM == 'linux' else 'dll'
+
+SP_VDF1 = 'addons/source-python'
+SP_VDF2 = 'addons/source-python2'
+
 DATA_ZIP_FILE = DATA_PATH / 'source-python-data.zip'
+UPDATE_ZIP_FILE = UPDATE_PATH / 'source-python.zip'
+VDF_FILE = ADDONS_PATH / 'source-python.vdf'
+LOADER_FILE = ADDONS_PATH / f'source-python.{BINARY_EXT}'
+LOADER_UPDATE_FILE = UPDATE_PATH / 'addons' / f'source-python.{BINARY_EXT}'
+VDF_UPDATE_FILE = UPDATE_PATH / 'addons' / 'source-python.vdf'
+
 CHECKSUM_URL = 'http://data.sourcepython.com/checksum.txt'
 DATA_URL = 'http://data.sourcepython.com/source-python-data.zip'
+ARTIFACTS_URL = 'http://builds.sourcepython.com/job/Source.Python/lastSuccessfulBuild/api/json?tree=artifacts[relativePath]'
+BASE_DOWNLOAD_URL = 'http://builds.sourcepython.com/job/Source.Python/lastSuccessfulBuild/artifact/'
 
 
 # =============================================================================
@@ -62,12 +93,7 @@ def download_latest_data(timeout=3):
     :param float timeout:
         Number of seconds that need to pass until a timeout occurs.
     """
-    update_logger.log_debug('Downloading data to {} ...'.format(DATA_ZIP_FILE))
-    with urlopen(DATA_URL, timeout=timeout) as url:
-        data = url.read()
-
-    with DATA_ZIP_FILE.open('wb') as f:
-        f.write(data)
+    download_file(DATA_URL, DATA_ZIP_FILE, timeout)
 
 def unpack_data():
     """Unpack ``source-python-data.zip``."""
@@ -101,3 +127,114 @@ def is_new_data_available(timeout=3):
         return True
 
     return DATA_ZIP_FILE.read_hexhash('md5') != get_latest_data_checksum(timeout)
+
+def get_artifacts():
+    """Return the artifacts of the latest build."""
+    update_logger.log_debug('Getting artifacts...')
+    with urlopen(ARTIFACTS_URL) as url:
+        data = json.loads(url.read())
+
+        for d in data['artifacts']:
+            yield d['relativePath']
+
+def get_download_url(game=SOURCE_ENGINE_BRANCH):
+    """Get the latest download URL for a specific game."""
+    for relative_path in get_artifacts():
+        if f'-{game}-' in relative_path:
+            return BASE_DOWNLOAD_URL + relative_path
+
+    raise ValueError(f'Unable to find a download URL for game "{game}".')
+
+def download_file(url_path, file_path, timeout=3):
+    """Download a file from an URL to a specific file."""
+    update_logger.log_debug(f'Downloading file ({url_path}) to {file_path} ...')
+    now = time.time()
+
+    with urlopen(url_path, timeout=timeout) as url:
+        data = url.read()
+
+    with file_path.open('wb') as f:
+        f.write(data)
+
+    update_logger.log_info(
+        'File has been downloaded. Time elapsed: {:0.2f} seconds'.format(
+            time.time()-now))
+
+def clean_update_dir():
+    """Clear or create the update directory."""
+    if UPDATE_PATH.exists():
+        for f in UPDATE_PATH.listdir():
+            if f.isfile():
+                f.remove()
+            else:
+                f.rmtree()
+    else:
+        UPDATE_PATH.mkdir()
+
+def download_latest_version(timeout=3):
+    """Download the latest version."""
+    download_file(get_download_url(), UPDATE_ZIP_FILE, timeout)
+
+def apply_update_stage1():
+    """Apply stage 1 of the version update."""
+    update_logger.log_message('Applying Source.Python update stage 1...')
+
+    # Extract all files to the update directory
+    with ZipFile(UPDATE_ZIP_FILE) as zip:
+        zip.extractall(UPDATE_PATH)
+
+    UPDATE_ZIP_FILE.remove()
+    VDF_UPDATE_FILE.remove()
+
+    if PLATFORM == 'windows':
+        _apply_update_stage1_windows()
+    else:
+        _apply_update_stage1_linux()
+
+def _apply_update_stage1_windows():
+    """Apply the Windows specific part of stage 1.
+
+    On Windows files that are currently in use (``source-python.dll``) can't be
+    replaced. Thus, this function checks if ``source-python.vdf`` exists. If it
+    does, the new ``source-python.dll`` is copied to the addons directory with
+    a new name (``source-python2.dll``). After that the VDF entry is modified
+    to point to the new loader.
+    If ``source-python.vdf`` does not exist, manual action is required.
+    """
+    if not VDF_FILE.isfile():
+        update_logger.log_message(
+            f'Stage 1 has been applied. Please shutdown your server and move '
+            f'(do not copy) {LOADER_UPDATE_FILE} to {LOADER_FILE}. After that '
+            f'start your server to apply stage 2.')
+    else:
+        update_logger.log_debug('Determining current VDF entry...')
+        kv = KeyValues.load_from_file(VDF_FILE)
+
+        # Get the current and new entry for the VDF file
+        current_entry = kv.get_string('file')
+        if current_entry == SP_VDF2:
+            new_entry = SP_VDF1
+        elif current_entry == SP_VDF1:
+            new_entry = SP_VDF2
+        else:
+            raise ValueError(f'Unexpected entry in VDF: {current_entry}')
+
+        update_logger.log_debug(f'Current VDF entry: {current_entry}')
+        update_logger.log_debug(f'New VDF entry: {new_entry}')
+
+        update_logger.log_debug('Moving new loader binary to game directory...')
+        LOADER_UPDATE_FILE.move(GAME_PATH / f'{new_entry}.{BINARY_EXT}')
+
+        kv.set_string('file', new_entry)
+        kv.save_to_file(VDF_FILE)
+
+        update_logger.log_message(
+            'Stage 1 has been applied. Restart your server to apply stage 2.')
+
+def _apply_update_stage1_linux():
+    """Apply the Linux specific part of stage 1."""
+    update_logger.log_debug('Moving new loader binary to game directory...')
+    LOADER_UPDATE_FILE.move(LOADER_FILE)
+
+    update_logger.log_message(
+        'Stage 1 has been applied. Restart your server to apply stage 2.')

addons/source-python/packages/source-python/paths.py

@@ -13,7 +13,8 @@
 # =============================================================================
 # >> ALL DECLARATION
 # =============================================================================
-__all__ = ('BASE_PATH',
+__all__ = ('ADDONS_PATH',
+           'BASE_PATH',
            'CFG_PATH',
            'CUSTOM_DATA_PATH',
            'CUSTOM_PACKAGES_DOCS_PATH',
@@ -34,6 +35,7 @@
            'TRANSLATION_PATH',
            'AUTH_CFG_PATH',
            'BACKENDS_PATH',
+           'UPDATE_PATH'
            )
 
 
@@ -43,8 +45,14 @@
 # ../<game>
 GAME_PATH = Path(Path(__file__).rsplit('addons', 1)[0][:~0])
 
+# ../addons
+ADDONS_PATH = GAME_PATH / 'addons'
+
 # ../addons/source-python
-BASE_PATH = GAME_PATH / 'addons' / 'source-python'
+BASE_PATH = ADDONS_PATH / 'source-python'
+
+# ../addons/source-python/update
+UPDATE_PATH = BASE_PATH / 'update'
 
 # ../addons/source-python/docs
 DOCS_PATH = BASE_PATH / 'docs'

src/CMakeLists.txt

@@ -14,11 +14,13 @@ include("makefiles/shared.cmake")
 # ------------------------------------------------------------------
 Set(SOURCEPYTHON_LOADER_HEADERS
     loader/loader_main.h
+    loader/updater.h
     loader/definitions.h
 )
 
 Set(SOURCEPYTHON_LOADER_SOURCES
     loader/loader_main.cpp
+    loader/updater.cpp
 )
 
 Set(SOURCEPYTHON_LOADER_FILES

src/loader/loader_main.cpp

@@ -28,6 +28,7 @@
 // Source includes
 //---------------------------------------------------------------------------------
 #include "loader_main.h"
+#include "updater.h"
 #include "interface.h"
 #include "eiface.h"
 #include "strtools.h"
@@ -36,6 +37,7 @@
 #endif
 
 #include "../core/utilities/shared_utils.h"
+#include <exception>
 
 //---------------------------------------------------------------------------------
 // Disable warnings.
@@ -51,6 +53,7 @@
 // Interfaces.
 //---------------------------------------------------------------------------------
 ICvar* g_pCVar = NULL; // This is required for linux linking..
+IVEngineServer* engine = NULL;
 
 //
 // The plugin is a static singleton that is exported as an interface
@@ -162,7 +165,7 @@ bool CSourcePython::Load( CreateInterfaceFn interfaceFactory, CreateInterfaceFn
 {
 	Msg(MSG_PREFIX "Loading...\n");
 
-	IVEngineServer* engine = (IVEngineServer*)interfaceFactory(INTERFACEVERSION_VENGINESERVER, NULL);
+	engine = (IVEngineServer*)interfaceFactory(INTERFACEVERSION_VENGINESERVER, NULL);
 
 	// Was the IVEngineServer interface retrieved properly?
 	if (!engine)
@@ -184,6 +187,18 @@ bool CSourcePython::Load( CreateInterfaceFn interfaceFactory, CreateInterfaceFn
 	DevMsg(1, MSG_PREFIX "Game directory: %s\n", szGameDir);
 	GenerateSymlink(szGameDir);
 
+	if (UpdateAvailable())
+	{
+		try
+		{
+			ApplyUpdateStage2();
+		}
+		catch (const std::exception& e)
+		{
+			Msg(MSG_PREFIX "An error occured during update stage 2:\n%s\n", e.what());
+		}
+	}
+
 	// ------------------------------------------------------------------
 	// Load windows dependencies.
 	// ------------------------------------------------------------------