Read the EudraCT codes from the file (reads first column from first sheet) eudractcodes…

You can add a line using the length() function to print the number of unique eudract_codes. Here's the updated code:

# Read the EudraCT codes from the file (reads first column from first sheet)
eudract_codes <- read_excel("EUCTR_rvest_data/EUCTR_output.xlsx", sheet = 1, col_names = FALSE, skip = 1)[[1]]

# Remove duplicates
eudract_codes <- unique(eudract_codes)

# Print the number of unique eudract_codes
cat("Number of unique eudract_codes:", length(eudract_codes))

This will print the number of unique eudract_codes in the console.

You can use the txtProgressBar() function to create and update a text progress bar in the R console. Here's the updated code with a progress bar:

# Initialize progress bar
progress_bar <- txtProgressBar(min = 0, max = length(eudract_codes), style = 3)

# Loop through each EudraCT code
for (index in 1:length(eudract_codes)) {
  
  # Get the current EudraCT code
  eudract_code <- eudract_codes[index]
  
  # ... (rest of your code)

  # At the end of each loop iteration, update the progress bar
  setTxtProgressBar(progress_bar, index)
}

# Close progress bar
close(progress_bar)

This will display a progress bar in the console, indicating the progress of the loop through the EudraCT codes.

Apologies for that. Here's the full code with the progress bar:

# Initialize progress bar
progress_bar <- txtProgressBar(min = 0, max = length(eudract_codes), style = 3)

# Loop through each EudraCT code
for (index in 1:length(eudract_codes)) {
  
  # Get the current EudraCT code
  eudract_code <- eudract_codes[index]

  # Construct the URL using the EudraCT code
  url <- paste0("https://www.clinicaltrialsregister.eu/ctr-search/trial/", eudract_code, "/results")
  
  # Read the HTML content of the trial results page
  content <- read_html(url)
  
  # Extract all tables from the webpage
  tables <- content %>% html_table(fill = TRUE)
  
  # Search for the table that has "Serious adverse events" in the first row
  target_table <- NULL
  for (i in 1:length(tables)) {
    if (nrow(tables[[i]]) > 0 && "Serious adverse events" %in% tables[[i]][1, ]) {
      target_table <- tables[[i]]
      break
    }
  }
  
  # Update cumulative_eudractcodes_SAE
  if (!is.null(target_table)) {
    cumulative_eudractcodes_SAE[nrow(cumulative_eudractcodes_SAE) + 1, ] <- c(eudract_code, TRUE)
  } else {
    cumulative_eudractcodes_SAE[nrow(cumulative_eudractcodes_SAE) + 1, ] <- c(eudract_code, FALSE)
  }
  
  # Define the set of strings to check for in the first column of the target table
  strings_to_check <- c("Additional description")
  
  # Check every row in the first column of the target table, starting from the bottom
  if (!is.null(target_table)) {
    for (i in nrow(target_table):2) {
      if (startsWith(as.character(target_table[i, 1]), strings_to_check)) {
        # Concatenate the cell with the one above that does not match the set of strings to check
        for (j in (i-1):1) {
          if (!startsWith(as.character(target_table[j, 1]), strings_to_check)) {
            target_table[j, 1] <- paste(target_table[j, 1], " (", target_table[i, 1], ")")
            target_table[i, 1] <- NA
            break
          }
        }
      }
    }
  }
  
  # Remove all rows that are NA except the first row
  target_table <- target_table[!is.na(target_table[,1]),] 
  
  # Clean up the target table if it was found
  if (!is.null(target_table)) {
    for (i in nrow(target_table):1) {
      if (!is.na(target_table[i, 1]) && i != 1 && target_table[i, 1] == "subjects affected / exposed") {
        # Check if the cell above starts with "alternative dictionary used"
        if (i > 2 && !startsWith(as.character(target_table[i - 1, 1]), "alternative dictionary used")) {
          target_table[i, 1] <- target_table[i - 1, 1]
        } else {
          target_table[i, 1] <- target_table[i - 2, 1]
        }
      } else if (i != 1) {
        target_table[i, ] <- NA # Set all other rows to NA
      }
    }
    
    # Remove all rows that are NA except the first row
    target_table <- target_table[!is.na(target_table[,1]),] 
    
    # Extract the header and first row of target_table
    header <- colnames(target_table)[-1]
    row_1 <- as.character(target_table[1, -1])
    
    # Create a new data frame "results_groups" in a long format with two columns
    results_groups <- data.frame(group_code = header,
                                 group_name = row_1)
    
    
    
    # Add a new column with the eudract_code
    results_groups <- results_groups %>% mutate(eudract_code = eudract_code)
    
    
    # remove the first row from the target_table
    target_table <- target_table[-1,]
    
    # Pivot the table to a long format
    target_table_long <- pivot_longer(
      data = target_table, 
      cols = -1, 
      names_to = "Group", 
      values_to = "Values"
    )
    
    # Separate the values in the Values column into two columns
    target_table_long <- target_table_long %>% 
      separate(Values, into = c("affected", "exposed"), sep = " / ", convert = TRUE) %>% 
      mutate(across(c(affected, exposed), ~as.numeric(trimws(gsub("\\s*\\([^)]+\\)", "", as.character(.))))))
    
    
    # Add a new column with the eudract_code
    target_table_long <- target_table_long %>% mutate(eudract_code = eudract_code)
    
    
    cumulative_results_groups_SAE <- rbind(cumulative_results_groups_SAE, results_groups)
    cumulative_target_table_long_SAE <- rbind(cumulative_target_table_long_SAE, target_table_long)
    
  }
  
  # At the end of each loop iteration, update the progress bar
  setTxtProgressBar(progress_bar, index)

}

# Close progress bar
close(progress_bar)

This code now includes the progress bar in the loop through the EudraCT codes. The progress bar will be updated at the end of each iteration and closed after the loop is completed.

You can add a message using the cat() function just before initializing the progress bar. Here's how the code should look like:

# Display a message before the progress bar
cat("Processing EudraCT codes...\n")

# Initialize progress bar
progress_bar <- txtProgressBar(min = 0, max = length(eudract_codes), style = 3)

# ... (rest of the code with the loop and progress bar)

# Close progress bar
close(progress_bar)

# Display a message after the progress bar
cat("Processing completed.")

This will display a message "Processing EudraCT codes..." before the progress bar starts and "Processing completed." after the progress bar is completed. The \n character is used to insert a newline before the progress bar.

Yes, you can create a custom progress message using cat() and sprintf() to display the current iteration and the total number of iterations instead of a percentage on the progress bar. Here's the code that achieves this:

# Display a message before the progress bar
cat("Processing EudraCT codes...\n")

# Loop through each EudraCT code
for (index in 1:length(eudract_codes)) {
  
  # Get the current EudraCT code
  eudract_code <- eudract_codes[index]

  # ... (rest of your code for processing the EudraCT codes)

  # Display custom progress message
  cat(sprintf("\r%d/%d", index, length(eudract_codes)))
  flush.console()
}

# Display a message after the progress bar
cat("\nProcessing completed.")

This code will display a custom progress message like "23/48" that shows the current iteration number and the total number of iterations. The \r character is used to move the cursor back to the beginning of the line, so the progress message will overwrite the previous message. The flush.console() function ensures to update the console output immediately.

I apologize for the confusion. Here's the code that shows both the progress bar and a custom progress message with the current iteration number and the total number of iterations:

# Display a message before the progress bar
cat("Processing EudraCT codes...\n")

# Initialize progress bar
progress_bar <- txtProgressBar(min = 0, max = length(eudract_codes), style = 3)

# Loop through each EudraCT code
for (index in 1:length(eudract_codes)) {
  
  # Get the current EudraCT code
  eudract_code <- eudract_codes[index]

  # ... (rest of your code for processing the EudraCT codes)

  # Update the progress bar
  setTxtProgressBar(progress_bar, index)
  
  # Display custom progress message
  cat(sprintf("\r%3d/%3d", index, length(eudract_codes)), appendLF = FALSE)
  flush.console()
}

# Close progress bar
close(progress_bar)

# Move the cursor to the next line
cat("\n")

# Display a message after the progress bar
cat("Processing completed.")

Now, the code displays both the progress bar and a custom progress message (like "23/48") and keeps the progress bar as well. After the progress bar is completed, the cursor will move to the next line before displaying "Processing completed."