--- work/gitup-0.96/gitup.c 2021-09-05 19:58:01.000000000 +0300
+++ /home/titus/builds/gitup.c 2021-12-01 12:38:45.404988000 +0200
@@ -52,6 +52,7 @@
#include <string.h>
#include <unistd.h>
#include <zlib.h>
+#include <regex.h>
#define GITUP_VERSION "0.96"
#define BUFFER_UNIT_SMALL 4096
@@ -122,6 +123,7 @@
char *remote_data_file;
char *remote_history_file;
char **ignore;
+ regex_t **reg_ignore;
int ignores;
bool keep_pack_file;
bool use_pack_file;
@@ -381,11 +383,18 @@
return (false);
for (x = 0; x < session->ignores; x++) {
+ if(!session->reg_ignore[x]) {
ignore = session->ignore[x];
if (strncmp(path, ignore, strlen(ignore)) == 0)
return (true);
+ } else {
+ if(!regexec(session->reg_ignore[x],path,0,NULL,0)) {
+ printf("skipped %s with %s\n",path,session->ignore[x]);
+ return (true);
}
+ }
+ }
return (false);
}
@@ -511,6 +520,8 @@
prune_tree(session, full_path);
} else {
+
+ if(!ignore_file(session,full_path,IGNORE_DELETE))
if (remove(full_path) == -1)
err(EXIT_FAILURE,
"prune_tree: cannot remove %s",
@@ -3073,6 +3084,50 @@
}
+static void
+add_ignore(connector *session,const char *string)
+{
+ char temp[BUFFER_UNIT_SMALL];
+ int ret_temp, length;
+ regex_t reg_temp;
+
+ session->ignore = (char **)realloc(
+ session->ignore,
+ (session->ignores + 1)*sizeof(char *));
+
+ session->reg_ignore = (regex_t **)realloc(
+ session->reg_ignore,
+ (session->ignores + 1)*sizeof(regex_t *));
+ if (session->ignore == NULL || session->reg_ignore == NULL)
+ err(EXIT_FAILURE,
+ "load_config_section: malloc");
+ session->reg_ignore[session->ignores] = NULL;
+ snprintf(temp, sizeof(temp),
+ "%s",
+ string);
+
+ if (temp[0] != '/')
+ snprintf(temp, sizeof(temp),
+ "%s/%s",
+ session->path_target,
+ string);
+ if(string[0] == '%') {
+ ret_temp = regcomp(®_temp,string + 1,REG_EXTENDED);
+ if(ret_temp) {
+ warnx("warning: can't compile %s, ignoring\n",string + 1);
+ } else {
+ session->reg_ignore[session->ignores] = malloc(sizeof(regex_t));
+ if(!session->reg_ignore[session->ignores]) {
+ err(EXIT_FAILURE,
+ "load_config_section: malloc2");
+ }
+ memcpy(session->reg_ignore[session->ignores],®_temp,sizeof(regex_t));
+ }
+ }
+ length = session->ignores++;
+ session->ignore[length] = strdup(temp);
+
+}
/*
* load_config_section
*
@@ -3088,7 +3143,6 @@
char temp[BUFFER_UNIT_SMALL];
int integer, length = 0;
bool boolean, target, path, ignores;
-
its = ucl_object_iterate_new(section);
while ((pair = ucl_object_iterate_safe(its, true))) {
@@ -3154,27 +3208,7 @@
while ((ignore = ucl_object_iterate_safe(iti, true))) {
string = ucl_object_tostring(ignore);
-
- session->ignore = (char **)realloc(
- session->ignore,
- (session->ignores + 1)*sizeof(char *));
-
- if (session->ignore == NULL)
- err(EXIT_FAILURE,
- "load_config_section: malloc");
-
- snprintf(temp, sizeof(temp),
- "%s",
- string);
-
- if (temp[0] != '/')
- snprintf(temp, sizeof(temp),
- "%s/%s",
- session->path_target,
- string);
-
- length = session->ignores++;
- session->ignore[length] = strdup(temp);
+ add_ignore(session,string);
}
ucl_object_iterate_free(iti);
@@ -3541,6 +3575,7 @@
.remote_data_file = NULL,
.remote_history_file = NULL,
.ignore = NULL,
+ .reg_ignore = NULL,
.ignores = 0,
.commit_history = false,
.keep_pack_file = false,
@@ -3581,7 +3616,7 @@
/* Process the command line parameters. */
- while ((option = getopt(argc, argv, "C:cd:h:klrS:t:u:v:w:")) != -1) {
+ while ((option = getopt(argc, argv, "I:C:cd:h:klrS:t:u:v:w:")) != -1) {
switch (option) {
case 'C':
if (session.verbosity)
@@ -3589,6 +3624,9 @@
"# Configuration file: %s\n",
configuration_file);
break;
+ case 'I':
+ add_ignore(&session,optarg);
+ break;
case 'c':
session.clone = true;
break;
@@ -4011,9 +4049,10 @@
"important changes.\n%s#\n",
session.updating);
- for (x = 0; x < session.ignores; x++)
+ for (x = 0; x < session.ignores; x++) {
free(session.ignore[x]);
-
+ free(session.reg_ignore[x]);
+ }
free(session.ignore);
free(session.response);
free(session.object);
… I'll be circling back to disk performance after I get the memory footprint issues addressed. …
here is a patch that adds regex support
also adds -I ignore on command line (works like ignore in config file)
a string preceded with a percent sign (%) is considered a regex otherwise a path
i had to modify prune_tree to check for ignores, otherwise you had to ignore work work/port work/port/file to protect a modified file
the only test i've done is
./gitup -v1 -I "%gitup/work/.*/gitup.c$" ports
Diff:--- work/gitup-0.96/gitup.c 2021-09-05 19:58:01.000000000 +0300 +++ /home/titus/builds/gitup.c 2021-12-01 12:38:45.404988000 +0200 @@ -52,6 +52,7 @@ #include <string.h> #include <unistd.h> #include <zlib.h> +#include <regex.h> #define GITUP_VERSION "0.96" #define BUFFER_UNIT_SMALL 4096 @@ -122,6 +123,7 @@ char *remote_data_file; char *remote_history_file; char **ignore; + regex_t **reg_ignore; int ignores; bool keep_pack_file; bool use_pack_file; @@ -381,11 +383,18 @@ return (false); for (x = 0; x < session->ignores; x++) { + if(!session->reg_ignore[x]) { ignore = session->ignore[x]; if (strncmp(path, ignore, strlen(ignore)) == 0) return (true); + } else { + if(!regexec(session->reg_ignore[x],path,0,NULL,0)) { + printf("skipped %s with %s\n",path,session->ignore[x]); + return (true); } + } + } return (false); } @@ -511,6 +520,8 @@ prune_tree(session, full_path); } else { + + if(!ignore_file(session,full_path,IGNORE_DELETE)) if (remove(full_path) == -1) err(EXIT_FAILURE, "prune_tree: cannot remove %s", @@ -3073,6 +3084,50 @@ } +static void +add_ignore(connector *session,const char *string) +{ + char temp[BUFFER_UNIT_SMALL]; + int ret_temp, length; + regex_t reg_temp; + + session->ignore = (char **)realloc( + session->ignore, + (session->ignores + 1)*sizeof(char *)); + + session->reg_ignore = (regex_t **)realloc( + session->reg_ignore, + (session->ignores + 1)*sizeof(regex_t *)); + if (session->ignore == NULL || session->reg_ignore == NULL) + err(EXIT_FAILURE, + "load_config_section: malloc"); + session->reg_ignore[session->ignores] = NULL; + snprintf(temp, sizeof(temp), + "%s", + string); + + if (temp[0] != '/') + snprintf(temp, sizeof(temp), + "%s/%s", + session->path_target, + string); + if(string[0] == '%') { + ret_temp = regcomp(®_temp,string + 1,REG_EXTENDED); + if(ret_temp) { + warnx("warning: can't compile %s, ignoring\n",string + 1); + } else { + session->reg_ignore[session->ignores] = malloc(sizeof(regex_t)); + if(!session->reg_ignore[session->ignores]) { + err(EXIT_FAILURE, + "load_config_section: malloc2"); + } + memcpy(session->reg_ignore[session->ignores],®_temp,sizeof(regex_t)); + } + } + length = session->ignores++; + session->ignore[length] = strdup(temp); + +} /* * load_config_section * @@ -3088,7 +3143,6 @@ char temp[BUFFER_UNIT_SMALL]; int integer, length = 0; bool boolean, target, path, ignores; - its = ucl_object_iterate_new(section); while ((pair = ucl_object_iterate_safe(its, true))) { @@ -3154,27 +3208,7 @@ while ((ignore = ucl_object_iterate_safe(iti, true))) { string = ucl_object_tostring(ignore); - - session->ignore = (char **)realloc( - session->ignore, - (session->ignores + 1)*sizeof(char *)); - - if (session->ignore == NULL) - err(EXIT_FAILURE, - "load_config_section: malloc"); - - snprintf(temp, sizeof(temp), - "%s", - string); - - if (temp[0] != '/') - snprintf(temp, sizeof(temp), - "%s/%s", - session->path_target, - string); - - length = session->ignores++; - session->ignore[length] = strdup(temp); + add_ignore(session,string); } ucl_object_iterate_free(iti); @@ -3541,6 +3575,7 @@ .remote_data_file = NULL, .remote_history_file = NULL, .ignore = NULL, + .reg_ignore = NULL, .ignores = 0, .commit_history = false, .keep_pack_file = false, @@ -3581,7 +3616,7 @@ /* Process the command line parameters. */ - while ((option = getopt(argc, argv, "C:cd:h:klrS:t:u:v:w:")) != -1) { + while ((option = getopt(argc, argv, "I:C:cd:h:klrS:t:u:v:w:")) != -1) { switch (option) { case 'C': if (session.verbosity) @@ -3589,6 +3624,9 @@ "# Configuration file: %s\n", configuration_file); break; + case 'I': + add_ignore(&session,optarg); + break; case 'c': session.clone = true; break; @@ -4011,9 +4049,10 @@ "important changes.\n%s#\n", session.updating); - for (x = 0; x < session.ignores; x++) + for (x = 0; x < session.ignores; x++) { free(session.ignore[x]); - + free(session.reg_ignore[x]); + } free(session.ignore); free(session.response); free(session.object);
--- gitup.c.ORI 2021-12-07 09:47:08.645516000 +0100
+++ gitup.c 2021-12-07 09:47:02.493263000 +0100
@@ -514,6 +514,8 @@
"prune_tree: cannot stat() %s",
full_path);
+ if( ignore_file( session, full_path, IGNORE_DELETE ))
+ continue;
if (S_ISDIR(sb.st_mode) != 0) {
if ((entry->d_namlen == 1) && (strcmp(entry->d_name, "." ) == 0))
continue;
When using an ignore entry which matches a self created file (e.g. "sysutils/bareos17-server/lala")
and a subsequent gitup removes "sysutils/bareos17-server", our to-be-ignored file is removed
as well. This can be circumvented with the following patch (warning: copy and paste):
--- gitup.c.ORI 2021-12-07 09:47:08.645516000 +0100 +++ gitup.c 2021-12-07 09:47:02.493263000 +0100 @@ -514,6 +514,8 @@ "prune_tree: cannot stat() %s", full_path); + if( ignore_file( session, full_path, IGNORE_DELETE )) + continue; if (S_ISDIR(sb.st_mode) != 0) { if ((entry->d_namlen == 1) && (strcmp(entry->d_name, "." ) == 0)) continue;
Of course this results in a warning
" ! cannot remove /tmp/g/sysutils/bareos17-server"
and has to be dealt with manually but it's still better than silently deleting it.
I have an old (as of 2021/12/03) ports directory in /tmp/g. In my gitup.conf I have anI'm not having any luck recreating the scenario you've described. When I create custom directories and files and add them to the ignores list, they survive subsequent runs.
When I clone an old commit of the https://github.com/freebsd/freebsd-ci repository where scripts/build/config-11 still exists, add a custom directory scripts/build/config-11/stuff to it and add another file scripts/build/config-11/stuff/test, add "scripts/build/config-11/stuff" to the ignores list and then pull the latest commit, the contents of scripts/build/config-11 are removed except for the custom directory and file.
Could you please send me a specific case where the custom files are deleted? Thanks!
I'm still not having any luck reproducing the issue. I have a copy of the ports tree from commit b298802b56c2f85df0cc28426b8612ea01751bd8 on December 5 (which has the math/eigen2 port) and if I add "math/eigen2/bla" to the ignores list, executeI have an old (as of 2021/12/03) ports directory in /tmp/g. In my gitup.conf I have an
"ignores" entry including "math/eigen2/bla". And I have this file:
buildbox:/tmp>ll g/math/eigen2/bla
-rw-r----- 1 user wheel 2852 15 Dec 06:40 g/math/eigen2/bla
Now I run gitup (where the eigen2 port is gone) and my file gets deleted:
...
- /tmp/g/lang/ruby27/files/patch-include_ruby_ruby.h
- /tmp/g/math/eigen2
- /tmp/g/math/eigen2/Makefile
- /tmp/g/math/eigen2/distinfo
- /tmp/g/math/eigen2/pkg-descr
- /tmp/g/math/eigen2/pkg-plist
- /tmp/g/misc/freebsd-release-manifests/files/MANIFESTS/amd64-amd64-12.3-RC1
...
buildbox:/tmp>ll g/math/eigen2/bla
ls: g/math/eigen2/bla: No such file or directory
With my patch from above the file won't get touched (and the dir not deleted). This is with gitup
from ports, but including the fix from https://forums.freebsd.org/posts/542233.
So in order to reproduce:
1. checkout ports with gitup
2. tar it to some place
3. run gitup until you find some port getting deleted
4. restore from the tar
5. copy something to "category/deletedport/myfile"
6. add "category/deletedport/myfile" to "ignores"
7. run gitup
touch math/eigen2/bla
and then run gitup ports
to fetch the latest revision, the contents of math/eigen2 are removed except for math/eigen2/bla.add math/eigen2/bar/bla.txt and exclude the file bla.txt it will be erased because math/eigen2/bar comes first, it is not excluded and it is pruned (which deletes bla.txt)
-c
intended for cases where, for example, a repair fails? gitup -c release
(15:04, 549 in history) followed, not immediately, by gitup release
(15:20, 560) resulted in repair, and a prompt to re-run.root@mowa219-gjp4-vm-freebsd-13-zfs:~ # gitup -V
gitup version 0.96
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # freebsd-version -kru
13.0-RELEASE-p4
13.0-RELEASE-p4
13.0-RELEASE-p5
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # pkg -vv | grep -e url -e enabled
url : "pkg+http://pkg.FreeBSD.org/FreeBSD:13:amd64/latest",
enabled : yes,
url : "https://alpha.pkgbase.live/current/FreeBSD:13:amd64/latest",
enabled : no,
url : "file:///usr/local/poudriere/data/packages/13-default",
enabled : yes,
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # gitup -c release
# Scanning local repository...
# Host: git.freebsd.org
# Port: 443
# Repository Path: /src.git
# Target Directory: /usr/src
# Have: 2646dd665909e60a369015c17cb602515e6025dc
# Want: 2646dd665909e60a369015c17cb602515e6025dc
# Branch: releng/13.0
# Done.
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # gitup release
# Scanning local repository...
# Host: git.freebsd.org
# Port: 443
# Repository Path: /src.git
# Target Directory: /usr/src
# Have: 2646dd665909e60a369015c17cb602515e6025dc
# Want: 2646dd665909e60a369015c17cb602515e6025dc
# Branch: releng/13.0
# Done.
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # grep -v \# /etc/freebsd-update.conf | sort
Components src world kernel
IDSIgnorePaths /usr/share/man/cat
IDSIgnorePaths /usr/share/man/whatis
IDSIgnorePaths /var/db/locate.database
IDSIgnorePaths /var/log
IgnorePaths
KeyPrint 800651ef4b4c71c27e60786d7b487188970f4b4169cc055784e21eb71d410cc5
MergeChanges /etc/ /boot/device.hints
ServerName update.FreeBSD.org
UpdateIfUnmodified /etc/ /var/ /root/ /.cshrc /.profile
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # setenv PAGER cat
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # freebsd-update fetch install
Looking up update.FreeBSD.org mirrors... 2 mirrors found.
Fetching metadata signature for 13.0-RELEASE from update2.freebsd.org... done.
Fetching metadata index... done.
Inspecting system... done.
Preparing to download files... done.
No updates needed to update system to 13.0-RELEASE-p5.
No updates are available to install.
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # history -S
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # history | tail -n 21
556 15:16 etcupdate -B
…
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # history | tail -n 29
549 15:04 gitup -c release
550 15:04 setenv PAGER cat ; freebsd-update fetch install
551 15:10 cat /etc/freebsd-update.conf
552 15:11 ls -hl /etc/freebsd-update.conf
553 15:11 grep -v \# /etc/freebsd-update.conf
554 15:11 ls -hl /etc/freebsd-update.conf
555 15:11 grep -v \# /etc/freebsd-update.conf | sort
556 15:16 etcupdate -B
557 15:16 etcupdate -B
558 15:16 etcupdate -p
559 15:16 etcupdate resolve
560 15:20 gitup release
561 15:24 history | grep gitup
562 15:24 history -S
563 15:30 pkg install qterminal
564 15:33 gitup -v
565 15:33 gitup --version
566 15:33 man gitup
567 15:34 gitup -V
568 15:34 freebsd-version -kru
569 15:34 pkg -vv | grep -e url -e enabled
570 15:34 gitup -c release
571 15:41 gitup release
572 15:42 grep -v \# /etc/freebsd-update.conf | sort
573 15:43 setenv PAGER cat
574 15:43 freebsd-update fetch install
575 15:51 history -S
576 15:53 history | tail -n 21
577 15:54 history | tail -n 29
root@mowa219-gjp4-vm-freebsd-13-zfs:~ #
It's possible that I keyed Control-C during the first of the commands, although I can't imagine doing so.The linked post confirms that the first command ran to completion; Done.
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # gitup -c release # Scanning local repository...
# Host: git.freebsd.org
# Port: 443
# Repository Path: /src.git
# Target Directory: /usr/src
# Have: 2646dd665909e60a369015c17cb602515e6025dc
# Want: 2646dd665909e60a369015c17cb602515e6025dc
# Branch: releng/13.0
# Done.
root@mowa219-gjp4-vm-freebsd-13-zfs:~ # …
sh
and gitup
merit attention from the processor. As in, there could be a hung process somewhere. Try running # ps -ax
in a different terminal/tty to verify my assumptions.I don't have that old test tree anymore but I just created a new one and will see what happensI still can't reproduce it but I'm thinking this might be due to differences in the ordering of files returned by the opendir() and readdir() calls. I just committed aafbsd's patch (thank you!). How does it work for both of you now?
… an option to minimize its memory footprint.
gitup quarterly
Secondgitup: fetch_pack: malformed pack data:
: inappropriate file type or format