Added script for svn repo mirroring.

Updated deliv config files.
2018-09-24 10:53:59 +02:00
parent 6ef4f1e8a8
commit ffa5f06e3d
21 changed files with 377 additions and 4 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ windows_deliv/headers/
 windows_deliv/install/bin/
 windows_deliv/install/binaries/
 windows_deliv/wix
+repos/mirrors/svn/*/
--- a/README.md
+++ b/README.md
@@ -0,0 +1,65 @@
+How to build Windows deliveries ?
+=================================
+
+1) local subversion repositories
+--------------------------------
+First, one can build local mirror of eiffelstudio and private ise repositories.
+It speeds up the delivery, and this way, you are not subject to network issue.
+
+See repos/ folder.
+For eiffelstudio:
+```
+> cd repos
+> init_eiffelstudio_mirror.bat
+> svnsync_all_mirrors.bat
+```
+
+For ise
+First, you need to setup the ISEVPN to access the subversion repository at svn://svn.ise:3691/ise_svn , then
+```
+> init_ise_mirror.bat your_ise_username your_ise_password
+> svnsync_all_mirrors.bat
+```
+
+Note, the delivery scripts are setup to use those mirrors, at this location, using file:///..../svn/eiffelstudio location for instance.
+
+
+2) setup the build scripts
+--------------------------
+in `windows_deliv`, you will find a new set of scripts.
+Please look into `etc\` folder
+update `config.btm` to set the expected version, for instance
+```
+	set STUDIO_VERSION_MAJOR_MINOR=18.11
+```
+And there are a few variables to set, but it is recommended to add a new file `etc\machines\%COMPUTERNAME%.btm`  , and set the variables specific to your machine.
+This way, you can switch easily from a machine to another.
+You can also decide in those files, to use the real subversion repositories, instead of the local mirrors.
+
+
+in `etc\profiles\` you can have various delivery profiles, for win64, using vc100, or vc150, ...
+you can decide to use one of them, by setting the environment variable ISE_BUILD_NAME to one of them (use the filename without the extension).
+Otherwise it will use default values for ISE_C_COMPILER* variables.
+
+3) Requirements
+---------------
+- TCC/LE (https://jpsoft.com/products/tcc-le.html)
+  edit menu.bat to set the TCCLECMD variable.
+- expected VisualStudio, to compile C Code.
+- WiX: in `C:\Program Files (x86)\Windows Installer XML v3\bin\  ` see file `install/package.wixproj`.
+
+4) And finally, open the wanted Visual Studio prompt and run
+```
+> menu.bat
+```
+and you can choose 1 for the full delivery, or 7 only for the GPL versions.
+
+
+-----
+
+Personal notes from Jocelyn: 
+- in set_simple_aliases.btm, see the alias `share_delivery` , it is called when the delivery is ready to share it on ftp, or elsewhere. The current setup is copying the file in `es_build_deliv/nightly` folder. On my machine, I have a symbolic link to a dropbox shared folder, so once it is ready I can access it, and I also setup an IFTTT that notifies any change in that folder, and send a message on slack.
+
+
+
+
--- a/repos/_hooks/pre-commit.cmd
+++ b/repos/_hooks/pre-commit.cmd
@@ -0,0 +1,86 @@
+@echo off
+
+:: PRE-COMMIT HOOK
+::
+:: The pre-commit hook is invoked before a Subversion txn is
+:: committed.  Subversion runs this hook by invoking a program
+:: (script, executable, binary, etc.) named 'pre-commit' (for which
+:: this file is a template), with the following ordered arguments:
+::
+::   [1] REPOS-PATH   (the path to this repository)
+::   [2] TXN-NAME     (the name of the txn about to be committed)
+::
+::   [STDIN] LOCK-TOKENS ** the lock tokens are passed via STDIN.
+::
+::   If STDIN contains the line "LOCK-TOKENS:\n" (the "\n" denotes a
+::   single newline), the lines following it are the lock tokens for
+::   this commit.  The end of the list is marked by a line containing
+::   only a newline character.
+::
+::   Each lock token line consists of a URI-escaped path, followed
+::   by the separator character '|', followed by the lock token string,
+::   followed by a newline.
+::
+:: The default working directory for the invocation is undefined, so
+:: the program should set one explicitly if it cares.
+::
+:: If the hook program exits with success, the txn is committed; but
+:: if it exits with failure (non-zero), the txn is aborted, no commit
+:: takes place, and STDERR is returned to the client.   The hook
+:: program can use the 'svnlook' utility to help it examine the txn.
+::
+:: On a Unix system, the normal procedure is to have 'pre-commit'
+:: invoke other programs to do the real work, though it may do the
+:: work itself too.
+::
+::   ***  NOTE: THE HOOK PROGRAM MUST NOT MODIFY THE TXN, EXCEPT  ***
+::   ***  FOR REVISION PROPERTIES (like svn:log or svn:author).   ***
+::
+::   This is why we recommend using the read-only 'svnlook' utility.
+::   In the future, Subversion may enforce the rule that pre-commit
+::   hooks should not modify the versioned data in txns, or else come
+::   up with a mechanism to make it safe to do so (by informing the
+::   committing client of the changes).  However, right now neither
+::   mechanism is implemented, so hook writers just have to be careful.
+::
+:: Note that 'pre-commit' must be executable by the user(s) who will
+:: invoke it (typically the user httpd runs as), and that user must
+:: have filesystem-level permission to access the repository.
+::
+:: On a Windows system, you should name the hook program
+:: 'pre-commit.bat' or 'pre-commit.exe',
+:: but the basic idea is the same.
+::
+:: The hook program typically does not inherit the environment of
+:: its parent process.  For example, a common problem is for the
+:: PATH environment variable to not be set to its usual value, so
+:: that subprograms fail to launch unless invoked via absolute path.
+:: If you're having unexpected problems with a hook program, the
+:: culprit may be unusual (or missing) environment variables.
+:: 
+:: Here is an example hook script, for a Unix /bin/sh interpreter.
+:: For more examples and pre-written hooks, see those in
+:: the Subversion repository at
+:: http://svn.apache.org/repos/asf/subversion/trunk/tools/hook-scripts/ and
+:: http://svn.apache.org/repos/asf/subversion/trunk/contrib/hook-scripts/
+
+@ECHO OFF
+setlocal
+set REPOS=%1
+set TXN=%2
+
+svnlook author -t %TXN% %REPOS% | findstr svnsync > nul
+if %errorlevel% equ 0 (goto VALID_USER)
+goto FORBIDDEN_USER
+
+:VALID_USER
+
+goto :eof
+
+:FORBIDDEN_USER
+svnlook author -t %TXN% %REPOS% >&2
+echo Author not allowed, only svnsync is allowed. >&2
+goto ERROR_EXIT
+ 
+:ERROR_EXIT
+exit /b 1
--- a/repos/_hooks/pre-revprop-change.cmd
+++ b/repos/_hooks/pre-revprop-change.cmd
@@ -0,0 +1,120 @@
+@echo off
+::  PRE-REVPROP-CHANGE HOOK
+:: 
+::  The pre-revprop-change hook is invoked before a revision property
+::  is added, modified or deleted.  Subversion runs this hook by invoking
+::  a program (script, executable, binary, etc.) named 'pre-revprop-change'
+::  (for which this file is a template), with the following ordered
+::  arguments:
+:: 
+::    [1] REPOS-PATH   (the path to this repository)
+::    [2] REV          (the revision being tweaked)
+::    [3] USER         (the username of the person tweaking the property)
+::    [4] PROPNAME     (the property being set on the revision)
+::    [5] ACTION       (the property is being 'A'dded, 'M'odified, or 'D'eleted)
+:: 
+::    [STDIN] PROPVAL  ** the new property value is passed via STDIN.
+:: 
+::  If the hook program exits with success, the propchange happens; but
+::  if it exits with failure (non-zero), the propchange doesn't happen.
+::  The hook program can use the 'svnlook' utility to examine the 
+::  existing value of the revision property.
+:: 
+::  WARNING: unlike other hooks, this hook MUST exist for revision
+::  properties to be changed.  If the hook does not exist, Subversion 
+::  will behave as if the hook were present, but failed.  The reason
+::  for this is that revision properties are UNVERSIONED, meaning that
+::  a successful propchange is destructive;  the old value is gone
+::  forever.  We recommend the hook back up the old value somewhere.
+:: 
+::  On a Unix system, the normal procedure is to have 'pre-revprop-change'
+::  invoke other programs to do the real work, though it may do the
+::  work itself too.
+:: 
+::  Note that 'pre-revprop-change' must be executable by the user(s) who will
+::  invoke it (typically the user httpd runs as), and that user must
+::  have filesystem-level permission to access the repository.
+:: 
+::  On a Windows system, you should name the hook program
+::  'pre-revprop-change.bat' or 'pre-revprop-change.exe',
+::  but the basic idea is the same.
+:: 
+::  The hook program typically does not inherit the environment of
+::  its parent process.  For example, a common problem is for the
+::  PATH environment variable to not be set to its usual value, so
+::  that subprograms fail to launch unless invoked via absolute path.
+::  If you're having unexpected problems with a hook program, the
+::  culprit may be unusual (or missing) environment variables.
+::  
+::  Here is an example hook script, for a Unix /bin/sh interpreter.
+::  For more examples and pre-written hooks, see those in
+::  the Subversion repository at
+::  http://svn.apache.org/repos/asf/subversion/trunk/tools/hook-scripts/ and
+::  http://svn.apache.org/repos/asf/subversion/trunk/contrib/hook-scripts/
+
+@ECHO OFF
+setlocal
+set repos=%1
+set rev=%2
+set user=%3
+set propname=%4
+set action=%5
+
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+:: Only allow changes to svn:log. The author, date and other revision
+:: properties cannot be changed
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+if /I '%propname%'=='svn:svn-lock' goto VALID_PROPNAME
+if /I '%propname%'=='svn:sync-lock' goto VALID_PROPNAME
+if /I '%propname%'=='svn:sync-from-url' goto VALID_PROPNAME
+if /I '%propname%'=='svn:sync-from-uuid' goto VALID_PROPNAME
+if /I '%propname%'=='svn:sync-last-merged-rev' goto VALID_PROPNAME
+if /I '%propname%'=='svn:sync-currently-copying' goto VALID_PROPNAME
+if /I '%propname%'=='svn:date' goto VALID_PROPNAME
+if /I '%propname%'=='svn:author' goto VALID_PROPNAME
+if /I '%propname%'=='svn:log' goto VALID_PROPNAME
+goto ERROR_PROPNAME
+
+:VALID_PROPNAME
+echo Propname=%propname% is allowed >&2
+
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+:: Only allow modifications to svn:log (no addition/overwrite or deletion)
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+rem echo Check [action=%action%] >&2
+if /I '%action%'=='A' goto VALID_ACTION
+if /I '%action%'=='D' goto VALID_ACTION
+if /I '%action%'=='M' goto VALID_ACTION
+goto ERROR_ACTION
+
+:VALID_ACTION
+echo Action=%action% is allowed >&2
+ 
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+:: Make sure that the new svn:log message contains some text.
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+::set bIsEmpty=true
+::for /f "tokens=*" %%g in ('find /V ""') do (
+ ::set bIsEmpty=false
+::)
+::if '%bIsEmpty%'=='true' goto ERROR_LOG_EMPTY
+::
+:::VALID_LOG
+::echo Log accepted >&2
+ 
+goto :eof
+ 
+:ERROR_LOG_EMPTY
+echo Empty svn:log properties are not allowed. >&2
+goto ERROR_EXIT
+ 
+:ERROR_PROPNAME
+echo Only changes to svn:log revision properties are allowed [prop=%propname%]. >&2
+goto ERROR_EXIT
+ 
+:ERROR_ACTION
+echo Only modifications to svn:log revision properties are allowed [action=%action%]. >&2
+goto ERROR_EXIT
+ 
+:ERROR_EXIT
+exit /b 1
--- a/repos/config.bat
+++ b/repos/config.bat
@@ -0,0 +1,2 @@
+set REPO_ROOT=%~dp0mirrors
+set "REPO_ROOT=%REPO_ROOT:\=/%"
--- a/repos/init_eiffelstudio_mirror.bat
+++ b/repos/init_eiffelstudio_mirror.bat
@@ -0,0 +1 @@
+init_svn_mirror.bat eiffelstudio https://svn.eiffel.com/eiffelstudio 
--- a/repos/init_ise_mirror.bat
+++ b/repos/init_ise_mirror.bat
@@ -0,0 +1 @@
+init_svn_mirror.bat ise svn://svn.ise:3691/ise_svn
--- a/repos/init_svn_mirror.bat
+++ b/repos/init_svn_mirror.bat
@@ -0,0 +1,14 @@
+setlocal
+call %~dp0config.bat
+
+set REPO_NAME=%1
+set REPO_URL=%2
+
+set REPO_DIR=mirrors\svn\%1
+set REPO_LOCAL=file:///%REPO_ROOT%/svn/%REPO_NAME%
+
+rd /q/s %REPO_DIR%
+svnadmin create %REPO_DIR%
+copy _hooks\*.cmd %REPO_DIR%\hooks\.
+svnsync init --sync-username svnsync "%REPO_LOCAL%" %REPO_URL%
+echo svnsync sync "%REPO_LOCAL%" > mirrors\svn\sync_%REPO_NAME%.bat
--- a/repos/init_svn_private_mirror.bat
+++ b/repos/init_svn_private_mirror.bat
@@ -0,0 +1,16 @@
+setlocal
+%~dp0config.bat
+
+set REPO_NAME=%1
+set REPO_URL=%2
+set REPO_USERNAME=%3
+set REPO_USERPASS=%4
+
+set REPO_DIR=mirrors\svn\%1
+set REPO_LOCAL=file:///%REPO_ROOT%/svn/%REPO_NAME%
+
+rd /q/s %REPO_DIR%
+svnadmin create %REPO_DIR%
+copy _hooks\*.cmd %REPO_DIR%\hooks\.
+svnsync init --sync-username svnsync --source-username %REPO_USERNAME% --source-password %REPO_USERPASS% "%REPO_LOCAL%" %REPO_URL%
+echo svnsync sync --sync-username svnsync --source-username %REPO_USERNAME% --source-password %REPO_USERPASS% "%REPO_LOCAL%" > mirrors\svn\sync_%REPO_NAME%.bat
--- a/repos/mirrors/svn/sync_eiffelstudio.bat
+++ b/repos/mirrors/svn/sync_eiffelstudio.bat
@@ -0,0 +1,5 @@
+setlocal
+set LOCAL_REPO_ROOT=%~dp0
+set "LOCAL_REPO_ROOT=%LOCAL_REPO_ROOT:\=/%"
+
+svnsync sync "file:///%LOCAL_REPO_ROOT%eiffelstudio" --username svnsync
--- a/repos/mirrors/svn/sync_ise.bat
+++ b/repos/mirrors/svn/sync_ise.bat
@@ -0,0 +1,5 @@
+setlocal
+set LOCAL_REPO_ROOT=%~dp0
+set "LOCAL_REPO_ROOT=%LOCAL_REPO_ROOT:\=/%"
+
+svnsync sync "file:///%LOCAL_REPO_ROOT%ise" --username svnsync
--- a/repos/serve_eiffelstudio_3699.bat
+++ b/repos/serve_eiffelstudio_3699.bat
@@ -0,0 +1 @@
+start svnserve -d -r %~dp0mirrors\svn\eiffelstudio --listen-port 3699
--- a/repos/serve_ise_3698.bat
+++ b/repos/serve_ise_3698.bat
@@ -0,0 +1 @@
+start svnserve -d -r %~dp0mirrors\svn\ise --listen-port 3698
--- a/repos/svnsync_all_mirrors.bat
+++ b/repos/svnsync_all_mirrors.bat
@@ -0,0 +1,6 @@
+@echo off
+setlocal
+
+FOR /F "tokens=*" %%G IN ('dir /b mirrors\svn\*.bat') DO start mirrors\svn\%%G
+:: FOR /F "tokens=*" %%G IN ('dir /b mirrors\svn\*') DO svnsync_mirror.bat %%G
+
--- a/repos/svnsync_mirror.bat
+++ b/repos/svnsync_mirror.bat
@@ -0,0 +1,8 @@
+@echo off
+setlocal
+%~dp0config.bat
+set REPO_NAME=%1
+set REPO_LOCAL=file:///%REPO_ROOT%/svn/%REPO_NAME%
+start svnsync sync "%REPO_LOCAL%"
+
+endlocal
--- a/repos/svnsync_set_uuid.bat
+++ b/repos/svnsync_set_uuid.bat
@@ -0,0 +1,17 @@
+@echo off
+setlocal
+%~dp0config.bat
+set REPO_NAME=%1
+set REPO_UUID=%2
+set DEST_TMP=".tmp-svn-%REPO_NAME%.tmp"
+
+set REPO_LOCAL=file:///%REPO_ROOT%/svn/%REPO_NAME%
+
+echo SVN-fs-dump-format-version: 2 > %DEST_TMP%
+echo "" >> %DEST_TMP%
+echo UUID: $UUID >> %DEST_TMP%
+
+svnadmin load --force-uuid %CD%\mirrors\svn\%REPO_NAME% < %DEST_TMP%
+del "%DEST_TMP%"
+
+endlocal
--- a/repos/svnsync_unlock.bat
+++ b/repos/svnsync_unlock.bat
@@ -0,0 +1,9 @@
+@echo off
+setlocal
+%~dp0config.bat
+set REPO_NAME=%1
+set REPO_LOCAL=file:///%REPO_ROOT%/svn/%REPO_NAME%
+
+svn pdel --revprop -r 0 svn:svn-lock "%REPO_LOCAL%"
+
+endlocal
--- a/windows_deliv/etc/machines/JFIATPC.btm
+++ b/windows_deliv/etc/machines/JFIATPC.btm
@@ -26,13 +26,16 @@ iff not defined WORK_DIR then
 	set WORK_DIR=C:\DEV\deliv
 	set WORK_DIR=M:\deliv
 endiff
+
+set LOCAL_REPO_ROOT=%~dp0..\..\..\repos\mirrors
+set "LOCAL_REPO_ROOT=%LOCAL_REPO_ROOT:\=/%"
 iff not defined DEFAULT_PUBLIC_SVN then
-	set DEFAULT_PUBLIC_SVN=file:///C:/DEV/ise/repos/mirrors/svn/eiffelstudio/trunk
+	set DEFAULT_PUBLIC_SVN=file:///%LOCAL_REPO_ROOT%/svn/eiffelstudio/trunk
 rem 	set DEFAULT_PUBLIC_SVN=svn://192.168.1.11:3699/trunk
 endiff
 iff not defined DEFAULT_ISE_SVN then
 	set DEFAULT_ISE_SVN=svn://jfiat@svn.ise:3691/ise_svn/trunk
-	set DEFAULT_ISE_SVN=file:///C:/DEV/ise/repos/mirrors/svn/ise/trunk
+	set DEFAULT_ISE_SVN=file:///%LOCAL_REPO_ROOT%/svn/ise/trunk
 rem	set DEFAULT_ISE_SVN=svn://192.168.1.11:3698/trunk
 endiff

--- a/windows_deliv/etc/machines/WINPC.btm
+++ b/windows_deliv/etc/machines/WINPC.btm
@@ -23,13 +23,16 @@ iff not defined WORK_DIR then
 	set WORK_DIR=C:\DEV\deliv
 	rem set WORK_DIR=M:\deliv
 endiff
+
+set LOCAL_REPO_ROOT=%~dp0..\..\..\repos\mirrors
+set "LOCAL_REPO_ROOT=%LOCAL_REPO_ROOT:\=/%"
 iff not defined DEFAULT_PUBLIC_SVN then
-	set DEFAULT_PUBLIC_SVN=file:///C:/DEV/ise/repos/mirrors/svn/eiffelstudio/trunk
+	set DEFAULT_PUBLIC_SVN=file:///%LOCAL_REPO_ROOT%/svn/eiffelstudio/trunk
 	rem set DEFAULT_PUBLIC_SVN=svn://192.168.1.11:3699/trunk
 endiff
 iff not defined DEFAULT_ISE_SVN then
 	set DEFAULT_ISE_SVN=svn://jfiat@svn.ise:3691/ise_svn/trunk
-	set DEFAULT_ISE_SVN=file:///C:/DEV/ise/repos/mirrors/svn/ise/trunk
+	set DEFAULT_ISE_SVN=file:///%LOCAL_REPO_ROOT%/svn/ise/trunk
 	rem set DEFAULT_ISE_SVN=svn://192.168.1.11:3698/trunk
 endiff

--- a/windows_deliv/etc/machines/local.btm
+++ b/windows_deliv/etc/machines/local.btm
@@ -10,6 +10,15 @@ iff not defined WORK_DIR then
 	set WORK_DIR=%~dp0..\..\_DELIV
 endiff

+set LOCAL_REPO_ROOT=%~dp0..\..\..\repos\mirrors
+set "LOCAL_REPO_ROOT=%LOCAL_REPO_ROOT:\=/%"
+iff not defined DEFAULT_PUBLIC_SVN then
+	set DEFAULT_PUBLIC_SVN=file:///%LOCAL_REPO_ROOT%/svn/eiffelstudio/trunk
+endiff
+iff not defined DEFAULT_ISE_SVN then
+	set DEFAULT_ISE_SVN=file:///%LOCAL_REPO_ROOT%/svn/ise/trunk
+endiff
+
 rem set ISE_EC_FLAGS=-full

 set DELIV_SKIP_ERROR="True"
--- a/windows_deliv/old_set_path.bat
+++ b/windows_deliv/old_set_path.bat
				`@@ -0,0 +1 @@`
				`init_svn_mirror.bat eiffelstudio https://svn.eiffel.com/eiffelstudio`
				`@@ -0,0 +1 @@`
				`init_svn_mirror.bat ise svn://svn.ise:3691/ise_svn`
				`@@ -0,0 +1 @@`
				`start svnserve -d -r %~dp0mirrors\svn\eiffelstudio --listen-port 3699`
				`@@ -0,0 +1 @@`
				`start svnserve -d -r %~dp0mirrors\svn\ise --listen-port 3698`