Merge pull request #505 from StoXProject/develop

Develop

Author: arnejohannesholmin

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: RstoxData
-Version: 2.2.1-9004
-Date: 2026-05-04
+Version: 2.2.1-9005
+Date: 2026-05-21
 Title: Tools to Read and Manipulate Fisheries Data
 Authors@R: c(
   person(given = "Arne Johannes", 

NEWS.md

@@ -1,3 +1,9 @@
+# RstoxData v2.2.1-9005  (2026-05-21)
+* Fixed bug in TranslateICESAcoustic() where translation of the tables Calibration, DataAcquisition, DataProcessing did not work due to error in the keys generated with expandICESKeysWithPrefix().
+* Changed ICESBiotic() to NOT set NA to 0 for SubsampleWeight in the Catch table, since the ICES acoustic database only accepts NA or a positive number as of the beginning of 2026.
+* In ICESBiotic() changed BioticData_NMDToICESBioticOne() to set SubsamplingFactor to catchweight / lengthsampleweight if both catchcount and lengthsamplecount are missing.
+
+
 # RstoxData v2.2.1-9004  (2026-05-05)
 * Fixed bug in StoxAcoustic where BeamKey was corrupted for data read by ReadAcoustic() from file in the ICESAcoustic format (LU25/LUF26 from LSSS). This resulted in NASC data being mixed between frequencies in the output from StoxAcoustic. The fix was to set sort = FALSE when merging in the Instrument table.
 * Fixed bug where TranslateICESAcoustic() did not manage to translate variables in the tables Instrument, Calibration, DataAcquisition and DataProcessing.

R/Filter.R

@@ -375,7 +375,9 @@ processFilter <- function(filters) {
 filterOneTable <- function(tableName, filter, data, treeStruct, propagateDownwards = TRUE, propagateUpwards = FALSE) {
 
 	# Special operators defined for filter operations. These are also supported in RstoxFramework:
+	# No longer needed since %notin% is now in the R base package, but still needed for oldreleases. We need to change this not in R 4.6 but in 4.7:
 	`%notin%` <- Negate(`%in%`)
+	# Why did we introduce this (in 2021)? Could it be for users who do not understand !=?
 	`%notequal%` <- function(x, table) is.na(x) | x %notin% table
 
 	# Declare the output

R/Read.R

@@ -54,6 +54,11 @@ check_landing_duplicates <- function(LandingData, warn=T, fix=F){
 #' 
 #' @param FileNames The paths of the biotic files.
 #' 
+#' @details
+#' The file is read using the ICESBiotic XSD defined by ICES acoustic database (https://www.ices.dk/data/data-portals/Pages/acoustic.aspx), with additional information about headers defined by the internal function \code{icesAcousticPreprocess}.
+#' 
+#' One particular case is when headers (e.g. LocalID) are stored as NA in the file. This results in the string "NA" and not the missing value NA. To read fields as NA, the field must be left out in the file.
+#' 
 #' @return
 #' An object of StoX data type BioticData: A list of a list of data.tables of the different levels of the input biotic files.
 #' 
@@ -96,6 +101,11 @@ ReadBiotic <- function(FileNames = character()) {
 #' 
 #' @param FileNames The paths of the acoustic files.
 #' 
+#' @details
+#' The file is read using the ICESAcoustic XSD defined by ICES acoustic database (https://www.ices.dk/data/data-portals/Pages/acoustic.aspx), with additional information about headers defined by the internal function \code{icesAcousticPreprocess}.
+#' 
+#' One particular case is when headers (e.g. LocalID) are stored as NA in the file. This results in the string "NA" and not the missing value NA. To read fields as NA, the field must be left out in the file.
+#' 
 #' @return
 #' An object of StoX data type AcousticData: A list of a list of data.tables of the different levels of the input acoustic files.
 #' 

R/StoxExport.R

@@ -742,11 +742,17 @@ BioticData_NMDToICESBioticOne <- function(
 		SpeciesCategoryNumber = catchcount,
 		WeightUnit = "kg", # Always kg in NMDBiotic (see http://www.imr.no/formats/nmdbiotic/)
 		SpeciesCategoryWeight = catchweight,
+		# At IMR we do not split catches by sex, but record sex at the individual level:
 		SpeciesSex = NA_character_,
 
-		#SubsampledNumber = lengthsamplecount,
+		# Samples:
 		SubsampledNumber = lengthsamplecount,
-		SubsamplingFactor = catchcount / lengthsamplecount,
+		# Prioritize count before weight, since before only catchcount / lengthsamplecount was used here:
+		#SubsamplingFactor = catchcount / lengthsamplecount,
+		SubsamplingFactor = ifelse(!is.na(lengthsamplecount) | is.na(catchcount), catchcount / lengthsamplecount, catchweight / lengthsampleweight),
+		
+		
+		ifelse(is.na(lengthsamplecount) | is.na(catchcount), catchweight / lengthsampleweight, catchcount / lengthsamplecount),
 		#SubsamplingFactor = ifelse(is.na(lengthsampleweight), 1, catchcount / lengthsamplecount),
 		SubsampleWeight = lengthsampleweight,
 		#SubsampleWeight = ifelse(is.na(lengthsampleweight), catchweight, lengthsampleweight),
@@ -763,8 +769,9 @@ BioticData_NMDToICESBioticOne <- function(
 	Catch[is.na(SpeciesCategoryNumber) & is.na(SpeciesCategoryWeight) & !is.na(SubsampledNumber), SpeciesCategoryNumber := SubsampledNumber]
 	Catch[is.na(SpeciesCategoryNumber) & is.na(SpeciesCategoryWeight) & !is.na(SubsampleWeight),  SpeciesCategoryWeight := SubsampleWeight]
 
-	# NA means that nothing is subsampled
-	Catch[!is.na(SpeciesCategoryWeight) & is.na(SubsampleWeight), SubsampleWeight := 0]
+	# This violated the requirement of the ICES Acoustic database that SubsampleWeight cannot be 0 (only missing or positive):
+	# # NA means that nothing is subsampled
+	# Catch[!is.na(SpeciesCategoryWeight) & is.na(SubsampleWeight), SubsampleWeight := 0]
 
 
 	# Combine required tables for the Biology level
@@ -836,7 +843,6 @@ BioticData_NMDToICESBioticOne <- function(
 
 
 
-
 #' Write ICESBiotic to CSV fille
 #'
 #' Writes \code{\link{ICESBioticData}} to a csv file for each input acoustic file used to create the \code{\link{ICESBioticData}}

R/Utilities.R

@@ -1210,17 +1210,54 @@ asIntegerAfterRound <- function(x, prec = sqrt(.Machine$double.eps)) {
 
 expandICESKeysWithPrefix <- function(ICESKeys) {
 
+	
 	# Declare the output:
 	ICESKeysOut <- ICESKeys
 
-	# Loop through the tables in reversed order, and paste the table name to the keys, overwriting as we move to the higher tables:
-	reversedTableOrder <- rev(names(ICESKeys))
-	for(tableName1 in reversedTableOrder) {
-		for(tableName2 in reversedTableOrder) {
-			ICESKeysOut[[tableName1]][ICESKeys[[tableName1]] %in% ICESKeys[[tableName2]]] <- paste0(tableName2, ICESKeys[[tableName1]][ICESKeys[[tableName1]] %in% ICESKeys[[tableName2]]])
+	# We use a translation table to do the renaming of the keys:
+	translation <- NULL
+	
+	# Loop through the tables:
+	tableNames <- names(ICESKeys)
+	for(tableName in tableNames) {
+		
+		# If the table only contains one key, prefix with the table name and overwrite the translation table:
+		if(length(ICESKeys[[tableName]]) == 1) {
+			
+			# Set the key directly, without use of the translation table here: 
+			newName <- paste0(tableName, ICESKeys[[tableName]])
+			ICESKeysOut[[tableName]] <- newName
+			
+			# Overwrite the translation table:
+			translation <- data.table(
+				from = ICESKeys[[tableName]], 
+				to = newName
+			)
+		}
+		# For other tables add the table name prefixed keys to the translation table and then perform the actual translation:
+		else {
+			
+			# Identify the keys that are not already fixed in the translation table:
+			areNotAlreadyPrepended <- ! ICESKeys[[tableName]] %in% translation$from
+			notAlreadyPrepended <- ICESKeys[[tableName]][areNotAlreadyPrepended]
+			newName <- paste0(tableName, notAlreadyPrepended)
+			
+			# Add to the translation table:
+			translation <- rbind(
+				translation,
+				data.table(
+					from = ICESKeys[[tableName]][areNotAlreadyPrepended], 
+					to = newName
+				)
+			)
+			
+			# Do the translation:
+			at <- match(ICESKeys[[tableName]], translation$from)
+			ICESKeysOut[[tableName]] <- translation$to[at]
 		}
 	}
 
+	
 	return(ICESKeysOut)
 }