[pacman-dev,v2] Add config option to specify amount of parallel download streams

Message ID 20200306021719.495142-1-anatol.pomozov@gmail.com
State New
Headers show
Series
  • [pacman-dev,v2] Add config option to specify amount of parallel download streams
Related show

Commit Message

Anatol Pomozov March 6, 2020, 2:17 a.m. UTC
It includes pacman.conf new 'ParallelDownloads' option that
specifies how many concurrent downloads cURL starts in parallel.

Add alpm_option_set_parallel_downloads() ALPM function that
allows to set this config option programmatically.

Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com>
---
 doc/pacman.conf.5.asciidoc |  5 ++++
 etc/pacman.conf.in         |  1 +
 lib/libalpm/alpm.h         |  5 ++++
 lib/libalpm/handle.c       | 15 ++++++++++
 lib/libalpm/handle.h       |  1 +
 src/pacman/conf.c          | 57 +++++++++++++++++++++++++++++++++++++-
 src/pacman/conf.h          |  2 ++
 7 files changed, 85 insertions(+), 1 deletion(-)

Comments

Allan McRae March 8, 2020, 5:14 a.m. UTC | #1
On 6/3/20 12:17 pm, Anatol Pomozov wrote:
> It includes pacman.conf new 'ParallelDownloads' option that
> specifies how many concurrent downloads cURL starts in parallel.
>
> Add alpm_option_set_parallel_downloads() ALPM function that
> allows to set this config option programmatically.
>
> Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com>
> ---
>  doc/pacman.conf.5.asciidoc |  5 ++++
>  etc/pacman.conf.in         |  1 +
>  lib/libalpm/alpm.h         |  5 ++++
>  lib/libalpm/handle.c       | 15 ++++++++++
>  lib/libalpm/handle.h       |  1 +
>  src/pacman/conf.c          | 57 +++++++++++++++++++++++++++++++++++++-
>  src/pacman/conf.h          |  2 ++
>  7 files changed, 85 insertions(+), 1 deletion(-)
>
> diff --git a/doc/pacman.conf.5.asciidoc b/doc/pacman.conf.5.asciidoc
> index b297e332..2c565efc 100644
> --- a/doc/pacman.conf.5.asciidoc
> +++ b/doc/pacman.conf.5.asciidoc
> @@ -205,6 +205,11 @@ Options
>  	Disable defaults for low speed limit and timeout on downloads. Use this
>  	if you have issues downloading files with proxy and/or security gateway.
>
> +*ParallelDownloads*::
> +	Specifies number of concurrent download streams. The value needs to be a
> +	positive integer. If this config option is not set then only one
download
> +	stream is used (i.e. download happens sequentially).
downloads happen

> +
>
>  Repository Sections
>  -------------------
> diff --git a/etc/pacman.conf.in b/etc/pacman.conf.in
> index 7446944f..44cea408 100644
> --- a/etc/pacman.conf.in
> +++ b/etc/pacman.conf.in
> @@ -34,6 +34,7 @@ Architecture = auto
>  #TotalDownload
>  CheckSpace
>  #VerbosePkgLists
> +ParallelDownloads = 5
>
>  # PGP signature checking
>  #SigLevel = Optional
> diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h
> index c2a069ad..93b97f44 100644
> --- a/lib/libalpm/alpm.h
> +++ b/lib/libalpm/alpm.h
> @@ -902,6 +902,11 @@ int
alpm_option_get_remote_file_siglevel(alpm_handle_t *handle);
>  int alpm_option_set_remote_file_siglevel(alpm_handle_t *handle, int
level);
>
>  int alpm_option_set_disable_dl_timeout(alpm_handle_t *handle,
unsigned short disable_dl_timeout);
> +/** Sets number of parallel streams to download database and package
files.
> + * If the function is not called then the default value of '1' stream
> + * (i.e. sequential download) is used.
Add:

 * @param handle the context handle
 * @param num_streams number of parallel download streams
 * @return 0 on success, -1 or error
 */

> + */> +int alpm_option_set_parallel_downloads(alpm_handle_t *handle,
unsigned int streams_num);
num_streams reads more clearly  (and throughout)

>
>  /** @} */
>
> diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c
> index fc7c1faf..fef1e5ad 100644
> --- a/lib/libalpm/handle.c
> +++ b/lib/libalpm/handle.c
> @@ -856,3 +856,18 @@ int SYMEXPORT
alpm_option_set_disable_dl_timeout(alpm_handle_t *handle,
>  #endif
>  	return 0;
>  }
> +
> +int SYMEXPORT alpm_option_set_parallel_downloads(alpm_handle_t *handle,
> +		unsigned int streams_num)
> +{
> +	CHECK_HANDLE(handle, return -1);
> +#ifdef HAVE_LIBCURL
> +	if(streams_num < 1) {
> +		return -1;
> +	}
> +	handle->parallel_downloads = streams_num;
> +#else
> +	(void)streams_num; /* silence unused variable warnings */
> +#endif
> +	return 0;
> +}
> \ No newline at end of file
Add a newline at the end of the file.

> diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h
> index c343f6e0..cd7104f9 100644
> --- a/lib/libalpm/handle.h
> +++ b/lib/libalpm/handle.h
> @@ -61,6 +61,7 @@ struct __alpm_handle_t {
>  	/* libcurl handle */
>  	CURL *curl;             /* reusable curl_easy handle */
>  	unsigned short disable_dl_timeout;
> +	unsigned int parallel_downloads; /* number of download streams */
>  #endif
>
>  #ifdef HAVE_LIBGPGME
> diff --git a/src/pacman/conf.c b/src/pacman/conf.c
> index f9de386f..8fa7f94b 100644
> --- a/src/pacman/conf.c
> +++ b/src/pacman/conf.c
> @@ -110,7 +110,8 @@ config_t *config_new(void)
>  		newconfig->localfilesiglevel = ALPM_SIG_USE_DEFAULT;
>  		newconfig->remotefilesiglevel = ALPM_SIG_USE_DEFAULT;
>  	}
> -
Keep the new line.

> +	/* by default use 1 download stream */
> +	newconfig->parallel_downloads = 1;
>  	newconfig->colstr.colon   = ":: ";
>  	newconfig->colstr.title   = "";
>  	newconfig->colstr.repo    = "";
> @@ -400,6 +401,32 @@ int config_set_arch(const char *arch)
>  	return 0;
>  }
>
> +/**
> + * Parse a string into long number. The input string has to be non-empty
> + * and represent a number that fits long type.
> + * @param value the string to parse
> + * @param result pointer to long where the final result will be stored.
> + *   This result is modified iff the input string parsed sucessfully.
if
successfully

> + * @return 0 in case if value parsed sucessfully, 1 otherwise.
successfully

> + */
> +static int parse_number(char *value, long *result) {
> +	char *endptr;
> +	long val;
> +	int invalid;
> +
> +	errno = 0; /* To distinguish success/failure after call */
> +	val = strtol(value, &endptr, 10);
> +	invalid = (errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
> +		|| (*endptr != '\0')
> +		|| (endptr == value);
> +
> +	if(!invalid) {
> +		*result = val;
> +	}
> +
> +	return invalid;
> +}
> +
OK

Rest of patch is fine.

A
Anatol Pomozov March 8, 2020, 8:32 p.m. UTC | #2
Hi

On Sat, Mar 7, 2020 at 9:56 PM Allan McRae <allan@archlinux.org> wrote:
>
> On 6/3/20 12:17 pm, Anatol Pomozov wrote:
> > It includes pacman.conf new 'ParallelDownloads' option that
> > specifies how many concurrent downloads cURL starts in parallel.
> >
> > Add alpm_option_set_parallel_downloads() ALPM function that
> > allows to set this config option programmatically.
> >
> > Signed-off-by: Anatol Pomozov <anatol.pomozov@gmail.com>
> > ---
> >  doc/pacman.conf.5.asciidoc |  5 ++++
> >  etc/pacman.conf.in         |  1 +
> >  lib/libalpm/alpm.h         |  5 ++++
> >  lib/libalpm/handle.c       | 15 ++++++++++
> >  lib/libalpm/handle.h       |  1 +
> >  src/pacman/conf.c          | 57 +++++++++++++++++++++++++++++++++++++-
> >  src/pacman/conf.h          |  2 ++
> >  7 files changed, 85 insertions(+), 1 deletion(-)
> >
> > diff --git a/doc/pacman.conf.5.asciidoc b/doc/pacman.conf.5.asciidoc
> > index b297e332..2c565efc 100644
> > --- a/doc/pacman.conf.5.asciidoc
> > +++ b/doc/pacman.conf.5.asciidoc
> > @@ -205,6 +205,11 @@ Options
> >       Disable defaults for low speed limit and timeout on downloads. Use this
> >       if you have issues downloading files with proxy and/or security gateway.
> >
> > +*ParallelDownloads*::
> > +     Specifies number of concurrent download streams. The value needs to be a
> > +     positive integer. If this config option is not set then only one
> download
> > +     stream is used (i.e. download happens sequentially).
> downloads happen

done

>
> > +
> >
> >  Repository Sections
> >  -------------------
> > diff --git a/etc/pacman.conf.in b/etc/pacman.conf.in
> > index 7446944f..44cea408 100644
> > --- a/etc/pacman.conf.in
> > +++ b/etc/pacman.conf.in
> > @@ -34,6 +34,7 @@ Architecture = auto
> >  #TotalDownload
> >  CheckSpace
> >  #VerbosePkgLists
> > +ParallelDownloads = 5
> >
> >  # PGP signature checking
> >  #SigLevel = Optional
> > diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h
> > index c2a069ad..93b97f44 100644
> > --- a/lib/libalpm/alpm.h
> > +++ b/lib/libalpm/alpm.h
> > @@ -902,6 +902,11 @@ int
> alpm_option_get_remote_file_siglevel(alpm_handle_t *handle);
> >  int alpm_option_set_remote_file_siglevel(alpm_handle_t *handle, int
> level);
> >
> >  int alpm_option_set_disable_dl_timeout(alpm_handle_t *handle,
> unsigned short disable_dl_timeout);
> > +/** Sets number of parallel streams to download database and package
> files.
> > + * If the function is not called then the default value of '1' stream
> > + * (i.e. sequential download) is used.
> Add:
>
>  * @param handle the context handle
>  * @param num_streams number of parallel download streams
>  * @return 0 on success, -1 or error
>  */
>
> > + */> +int alpm_option_set_parallel_downloads(alpm_handle_t *handle,
> unsigned int streams_num);
> num_streams reads more clearly  (and throughout)

done

>
> >
> >  /** @} */
> >
> > diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c
> > index fc7c1faf..fef1e5ad 100644
> > --- a/lib/libalpm/handle.c
> > +++ b/lib/libalpm/handle.c
> > @@ -856,3 +856,18 @@ int SYMEXPORT
> alpm_option_set_disable_dl_timeout(alpm_handle_t *handle,
> >  #endif
> >       return 0;
> >  }
> > +
> > +int SYMEXPORT alpm_option_set_parallel_downloads(alpm_handle_t *handle,
> > +             unsigned int streams_num)
> > +{
> > +     CHECK_HANDLE(handle, return -1);
> > +#ifdef HAVE_LIBCURL
> > +     if(streams_num < 1) {
> > +             return -1;
> > +     }
> > +     handle->parallel_downloads = streams_num;
> > +#else
> > +     (void)streams_num; /* silence unused variable warnings */
> > +#endif
> > +     return 0;
> > +}
> > \ No newline at end of file
> Add a newline at the end of the file.

done

>
> > diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h
> > index c343f6e0..cd7104f9 100644
> > --- a/lib/libalpm/handle.h
> > +++ b/lib/libalpm/handle.h
> > @@ -61,6 +61,7 @@ struct __alpm_handle_t {
> >       /* libcurl handle */
> >       CURL *curl;             /* reusable curl_easy handle */
> >       unsigned short disable_dl_timeout;
> > +     unsigned int parallel_downloads; /* number of download streams */
> >  #endif
> >
> >  #ifdef HAVE_LIBGPGME
> > diff --git a/src/pacman/conf.c b/src/pacman/conf.c
> > index f9de386f..8fa7f94b 100644
> > --- a/src/pacman/conf.c
> > +++ b/src/pacman/conf.c
> > @@ -110,7 +110,8 @@ config_t *config_new(void)
> >               newconfig->localfilesiglevel = ALPM_SIG_USE_DEFAULT;
> >               newconfig->remotefilesiglevel = ALPM_SIG_USE_DEFAULT;
> >       }
> > -
> Keep the new line.
done

>
> > +     /* by default use 1 download stream */
> > +     newconfig->parallel_downloads = 1;
> >       newconfig->colstr.colon   = ":: ";
> >       newconfig->colstr.title   = "";
> >       newconfig->colstr.repo    = "";
> > @@ -400,6 +401,32 @@ int config_set_arch(const char *arch)
> >       return 0;
> >  }
> >
> > +/**
> > + * Parse a string into long number. The input string has to be non-empty
> > + * and represent a number that fits long type.
> > + * @param value the string to parse
> > + * @param result pointer to long where the final result will be stored.
> > + *   This result is modified iff the input string parsed sucessfully.
> if
> successfully
>
> > + * @return 0 in case if value parsed sucessfully, 1 otherwise.
> successfully
>
> > + */
> > +static int parse_number(char *value, long *result) {
> > +     char *endptr;
> > +     long val;
> > +     int invalid;
> > +
> > +     errno = 0; /* To distinguish success/failure after call */
> > +     val = strtol(value, &endptr, 10);
> > +     invalid = (errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
> > +             || (*endptr != '\0')
> > +             || (endptr == value);
> > +
> > +     if(!invalid) {
> > +             *result = val;
> > +     }
> > +
> > +     return invalid;
> > +}
> > +
> OK
>
> Rest of patch is fine.
>
> A

PTAL the upcoming patch.

Patch

diff --git a/doc/pacman.conf.5.asciidoc b/doc/pacman.conf.5.asciidoc
index b297e332..2c565efc 100644
--- a/doc/pacman.conf.5.asciidoc
+++ b/doc/pacman.conf.5.asciidoc
@@ -205,6 +205,11 @@  Options
 	Disable defaults for low speed limit and timeout on downloads. Use this
 	if you have issues downloading files with proxy and/or security gateway.
 
+*ParallelDownloads*::
+	Specifies number of concurrent download streams. The value needs to be a
+	positive integer. If this config option is not set then only one download
+	stream is used (i.e. download happens sequentially).
+
 
 Repository Sections
 -------------------
diff --git a/etc/pacman.conf.in b/etc/pacman.conf.in
index 7446944f..44cea408 100644
--- a/etc/pacman.conf.in
+++ b/etc/pacman.conf.in
@@ -34,6 +34,7 @@  Architecture = auto
 #TotalDownload
 CheckSpace
 #VerbosePkgLists
+ParallelDownloads = 5
 
 # PGP signature checking
 #SigLevel = Optional
diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h
index c2a069ad..93b97f44 100644
--- a/lib/libalpm/alpm.h
+++ b/lib/libalpm/alpm.h
@@ -902,6 +902,11 @@  int alpm_option_get_remote_file_siglevel(alpm_handle_t *handle);
 int alpm_option_set_remote_file_siglevel(alpm_handle_t *handle, int level);
 
 int alpm_option_set_disable_dl_timeout(alpm_handle_t *handle, unsigned short disable_dl_timeout);
+/** Sets number of parallel streams to download database and package files.
+ * If the function is not called then the default value of '1' stream
+ * (i.e. sequential download) is used.
+ */
+int alpm_option_set_parallel_downloads(alpm_handle_t *handle, unsigned int streams_num);
 
 /** @} */
 
diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c
index fc7c1faf..fef1e5ad 100644
--- a/lib/libalpm/handle.c
+++ b/lib/libalpm/handle.c
@@ -856,3 +856,18 @@  int SYMEXPORT alpm_option_set_disable_dl_timeout(alpm_handle_t *handle,
 #endif
 	return 0;
 }
+
+int SYMEXPORT alpm_option_set_parallel_downloads(alpm_handle_t *handle,
+		unsigned int streams_num)
+{
+	CHECK_HANDLE(handle, return -1);
+#ifdef HAVE_LIBCURL
+	if(streams_num < 1) {
+		return -1;
+	}
+	handle->parallel_downloads = streams_num;
+#else
+	(void)streams_num; /* silence unused variable warnings */
+#endif
+	return 0;
+}
\ No newline at end of file
diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h
index c343f6e0..cd7104f9 100644
--- a/lib/libalpm/handle.h
+++ b/lib/libalpm/handle.h
@@ -61,6 +61,7 @@  struct __alpm_handle_t {
 	/* libcurl handle */
 	CURL *curl;             /* reusable curl_easy handle */
 	unsigned short disable_dl_timeout;
+	unsigned int parallel_downloads; /* number of download streams */
 #endif
 
 #ifdef HAVE_LIBGPGME
diff --git a/src/pacman/conf.c b/src/pacman/conf.c
index f9de386f..8fa7f94b 100644
--- a/src/pacman/conf.c
+++ b/src/pacman/conf.c
@@ -110,7 +110,8 @@  config_t *config_new(void)
 		newconfig->localfilesiglevel = ALPM_SIG_USE_DEFAULT;
 		newconfig->remotefilesiglevel = ALPM_SIG_USE_DEFAULT;
 	}
-
+	/* by default use 1 download stream */
+	newconfig->parallel_downloads = 1;
 	newconfig->colstr.colon   = ":: ";
 	newconfig->colstr.title   = "";
 	newconfig->colstr.repo    = "";
@@ -400,6 +401,32 @@  int config_set_arch(const char *arch)
 	return 0;
 }
 
+/**
+ * Parse a string into long number. The input string has to be non-empty
+ * and represent a number that fits long type.
+ * @param value the string to parse
+ * @param result pointer to long where the final result will be stored.
+ *   This result is modified iff the input string parsed sucessfully.
+ * @return 0 in case if value parsed sucessfully, 1 otherwise.
+ */
+static int parse_number(char *value, long *result) {
+	char *endptr;
+	long val;
+	int invalid;
+
+	errno = 0; /* To distinguish success/failure after call */
+	val = strtol(value, &endptr, 10);
+	invalid = (errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
+		|| (*endptr != '\0')
+		|| (endptr == value);
+
+	if(!invalid) {
+		*result = val;
+	}
+
+	return invalid;
+}
+
 /**
  * Parse a signature verification level line.
  * @param values the list of parsed option values
@@ -677,6 +704,33 @@  static int _parse_options(const char *key, char *value,
 				return 1;
 			}
 			FREELIST(values);
+		} else if(strcmp(key, "ParallelDownloads") == 0) {
+			long number;
+			int err;
+
+			err = parse_number(value, &number);
+			if(err) {
+				pm_printf(ALPM_LOG_WARNING,
+						_("config file %s, line %d: invalid value for '%s' : '%s'\n"),
+						file, linenum, "ParallelDownloads", value);
+				return 1;
+			}
+
+			if(number < 1) {
+				pm_printf(ALPM_LOG_WARNING,
+						_("config file %s, line %d: value for '%s' has to be positive : '%s'\n"),
+						file, linenum, "ParallelDownloads", value);
+				return 1;
+			}
+
+			if(number > INT_MAX) {
+				pm_printf(ALPM_LOG_WARNING,
+						_("config file %s, line %d: value for '%s' is too large : '%s'\n"),
+						file, linenum, "ParallelDownloads", value);
+				return 1;
+			}
+
+			config->parallel_downloads = number;
 		} else {
 			pm_printf(ALPM_LOG_WARNING,
 					_("config file %s, line %d: directive '%s' in section '%s' not recognized.\n"),
@@ -845,6 +899,7 @@  static int setup_libalpm(void)
 	alpm_option_set_noextracts(handle, config->noextract);
 
 	alpm_option_set_disable_dl_timeout(handle, config->disable_dl_timeout);
+	alpm_option_set_parallel_downloads(handle, config->parallel_downloads);
 
 	for(i = config->assumeinstalled; i; i = i->next) {
 		char *entry = i->data;
diff --git a/src/pacman/conf.h b/src/pacman/conf.h
index d954e637..42cddfbb 100644
--- a/src/pacman/conf.h
+++ b/src/pacman/conf.h
@@ -115,6 +115,8 @@  typedef struct __config_t {
 	/* When downloading, display the amount downloaded, rate, ETA, and percent
 	 * downloaded of the total download list */
 	unsigned short totaldownload;
+	/* number of parallel download streams */
+	unsigned int parallel_downloads;
 	/* select -Sc behavior */
 	unsigned short cleanmethod;
 	alpm_list_t *holdpkg;