The repo now includes all files, plus the matlab code, that is stored…

… on the uni sydney server. The PIOLab repo is now 1:1 what is necessary to run the lap on any IELab server.

.directory

@@ -0,0 +1,7 @@
+[Dolphin]
+AdditionalInfo=3
+Timestamp=2020,3,9,7,49,7
+ViewMode=2
+
+[Settings]
+ShowDotFiles=true

IEfeeds_code/.directory

@@ -0,0 +1,4 @@
+[Dolphin]
+AdditionalInfo=3
+Timestamp=2020,1,9,18,18,6
+ViewMode=1

IEfeeds_code/IEsettings/IE_longnames.txt

@@ -0,0 +1 @@
+IE feed from R for various regions with 20 industries and 22 products

IEfeeds_code/IEsettings/IE_shortnames.txt

@@ -0,0 +1 @@
+Ind20Pro22v1

IEfeeds_code/Ind20Pro22v1_InitialEstimate.m

@@ -0,0 +1,49 @@
+function [RegMap,IndMap,ProdMap]=Ind20Pro22v1_InitialEstimate(handles)
+
+    disp('Launching initial estimate for the extended PIOT version 1');
+
+    % Write handles variables into mat-file for R to read
+    % Working directory
+    filename = [handles.processeddatadir,'WorkingDirectory4R.mat'];
+    out = handles.motherALANGdir;
+    save(filename,'out');
+
+    % Write region aggregator to file
+    filename = [handles.processeddatadir,'RegionAggFile4R.mat'];
+    out = handles.regionaggfile;
+    save(filename,'out');
+
+    % Region aggregator
+    RegMap = csvread(handles.regionagg);
+
+    if size(RegMap,1) > size(RegMap,2) % check orientation of aggregator
+        RegMap  = RegMap';
+    end
+
+    reg_proxy = ones(size(RegMap,2),1);
+    RegMap = prorate(RegMap,'col_proxy',reg_proxy);
+
+    % Product aggregator
+    ProdMap = csvread(handles.sectoraggprod);
+
+    if size(ProdMap,1) > size(ProdMap,2) % check orientation of aggregator
+        ProdMap  = ProdMap';
+    end
+
+    prod_proxy = ones(size(ProdMap,2),1);
+    ProdMap = prorate(ProdMap,'col_proxy',prod_proxy);
+
+    % Industry aggregator 
+    IndMap = csvread(handles.sectoragg);
+
+    if size(IndMap,1) > size(IndMap,2) % check orientation of aggregator
+        IndMap  = IndMap';
+    end
+
+    ind_proxy = ones(size(IndMap,2),1);
+    IndMap = prorate(IndMap,'col_proxy',ind_proxy);
+
+    command = 'Rscript /import/emily1/isa/IELab/Roots/PIOLab/Rscripts/IEfeeds_code/Ind20Pro22v1_InitialEstimate.R';
+    system(command,'-echo');
+
+    end    

IEfeeds_code/README.txt

@@ -1 +1 @@
-See \PIOLab\Rscripts\IEfeeds_code for IEfeeds_code.
+See \PIOLab\Rscripts\IEfeeds_code for IE feeds in R scripts.

IEfeeds_code/datafeed_PIOLab_InitialEstimate.m

@@ -0,0 +1,11 @@
+function [RegMap,IndMap,ProdMap] = datafeed_PIOLab_InitialEstimate(handles)
+
+% Master file for IE feeds
+
+    if handles.nonsurvey == 1
+
+       [RegMap,IndMap,ProdMap]=Ind20Pro22v1_InitialEstimate(handles);
+
+    end
+
+end

Rscripts/datafeeds_code/datafeed_PIOLab_BACI.R

@@ -16,6 +16,9 @@ if(Sys.info()[1] == "Linux"){
 source(paste0(root_folder,"Rscripts/Subroutines/InitializationR.R"))
 path["df_Processed"] <- paste0(path$Processed,"/",datafeed_name)
 
+# Load function to write arrays to files
+source(paste0(path$Subroutines,"/Numbers2File.R"))
+
 # Loading raw data
 source(paste0(path$Subroutines,"/Load_BACI.R"))
 
@@ -26,7 +29,7 @@ data <- SE_LogRegression(data,RSE$Minimum,RSE$Maximum)
 data <- select(data,-value)
 
 # Set variables
-reg_max <- nrow(root$region)
+n_reg <- nrow(root$region)
 n_yea <- as.character(year-2007)
 n_she <- "1"
 n_pro <- nrow(root$product)
@@ -36,45 +39,40 @@ n_ind <- nrow(root$industry)
 source(paste0(path$Subroutines,"/makeALANGheadline.R"))
 
 # Check if folder with processed data exists, in case delete and create empty one
-path_set <- paste0(path$root,"ProcessedData/",datafeed_name)
-if(dir.exists(path_set)) unlink(path_set,recursive = TRUE) 
-dir.create(path_set)
-
-mat <- matrix(0,nrow = n_pro,ncol = 1) # Empty matrix to put numbers in
+if(dir.exists(path$df_Processed)) unlink(path$df_Processed,recursive = TRUE) 
+dir.create(path$df_Processed)
 
 a <- 1 # Set starting value for alang line index
+
 for(i in unique(data$From)) # Loop over the exporting regions
 {
-  data_sel <- filter(data,From == i)  # Filter exporting region
+  sel <- list("data" = filter(data,From == i))  # Filter exporting region
 
-  for(j in unique(data_sel$To)) # Loop over trade partners
+  for(j in unique(sel$data$To)) # Loop over trade partners
   {
     # Add empty line with tag
     ALANG <- add_row(ALANG,'1' = paste0("DataFeed BACI from ",root$region$Name[i]," to ",root$region$Name[j]))
 
-    trader_sel <- filter(data_sel,To == j) %>% select(-From,-To) # select data for trade partners
-
-    values <- mat # Create empty column vector 
-    values[trader_sel$Product,1] <- trader_sel$Quantity  # Write values
+    # select data for trade partners:
+    sel[["trader"]] <- filter(sel$data,To == j) %>% select(Product,Quantity,SE) 
 
-    filename_value <- paste0("BACI_",year,"_Values_",root$region$Name[i],"-",
-                       root$region$Name[j],".csv")      # Set name of the file
-
-    write.table(values,row.names = FALSE,col.names = FALSE, sep = ",",
-                file = paste0(path_set,"/",filename_value))   # Write array to folder
-
-    SE <- mat # Create empty column vector and write SE
+    df <- data.frame("RHS" = rep(0,n_pro),"SE" = rep(0,n_pro)) # Create empty data frame
 
-    SE[trader_sel$Product,1] <- trader_sel$SE # Write SE numbers into array
+    df$RHS[sel$trader$Product] <- sel$trader$Quantity # Write values
+    df$SE[sel$trader$Product] <- sel$trader$SE # Write standard errors
 
-    filename_SE <- paste0("BACI_",year,"_SE_",root$region$Name[i],"-",
-                       root$region$Name[j],".csv")      # Set name of the file
+    # Set name of the files:
+    filename <- list("RHS" = paste0("/BACI/BACI_",year,"_RHS_",root$region$Name[i],
+                                    "-",root$region$Name[j],".csv"),
+                     "SE" = paste0("/BACI/BACI_",year,"_SE_",root$region$Name[i],"-",
+                                   root$region$Name[j],".csv")) 
 
-    write.table(SE,row.names = FALSE,col.names = FALSE, sep = ",",
-                file = paste0(path_set,"/",filename_SE))   # Write array to folder
+    # Write data to file:
+    Numbers2File(df$RHS,paste0(path$Processed,filename$RHS))
+    Numbers2File(df$SE,paste0(path$Processed,filename$SE))
 
-    ALANG$Value[a] <- paste0("DATAPATH/",filename_value)
-    ALANG$S.E.[a] <- paste0("DATAPATH/",filename_SE)
+    ALANG$Value[a] <- paste0("DATAPATH",filename$RHS)
+    ALANG$S.E.[a] <- paste0("DATAPATH",filename$SE)
 
     ALANG$`Row parent`[a] <- as.character(i)
     ALANG$`Column parent`[a] <- as.character(j)
@@ -97,17 +95,6 @@ ALANG$Margin <- n_she
 ALANG$`Row child` <- "2"
 ALANG$`Row grandchild` <- "1:e"
 ALANG$`Column child` <- "1"
-
-# # Create and write sector concordance to file
-# Concord <- matrix(1,nrow = n_pro,ncol = n_ind) # block matrix required for aggregation
-# 
-# # Set name and path to concordance and write to folder
-# Concorda_name <- "BACI_Sec_Concordance"
-# Concord_path <- paste0(path$Concordance,"/",Concorda_name,".csv")
-# write.table(Concord,file = Concord_path,row.names = FALSE,col.names = FALSE,sep = ",")
-
-# Add path to concordance to ALANG commands
-#ALANG$`Column grandchild` <- paste0("1-e t2 CONCPATH/",Concorda_name,".csv")
 ALANG$`Column grandchild` <- "1-e"
 
 # Call script that writes the ALANG file to the repsective folder in the root

Rscripts/datafeeds_code/datafeed_PIOLab_IRPexports.R

@@ -11,9 +11,15 @@ if(Sys.info()[1] == "Linux"){
   root_folder <- "/import/emily1/isa/IELab/Roots/PIOLab/"}else{
     root_folder <- "C:/Users/hwieland/Github workspace/PIOLab/"}
 ################################################################################
+
 # Initializing R script (load R packages and set paths to folders etc.)
 source(paste0(root_folder,"Rscripts/Subroutines/InitializationR.R"))
 
+path["df_Processed"] <- paste0(path$Processed,"/",datafeed_name)
+
+# Load function to write arrays to files
+source(paste0(path$Subroutines,"/Numbers2File.R"))
+
 # Loading raw data
 data <- read.csv(paste0(path$Raw,"/IRP/all_CCC_Exp_ResearchDB.csv"),stringsAsFactors=FALSE)
 colnames(data)[6:49] <- 1970:2013
@@ -36,7 +42,7 @@ data <- data %>% filter(!Code %in% c(233,242))
 
 data <- data[!is.na(data$Code),] # Mayotte and Farour Isl. are not in root (tbc)
 
-reg_max <- nrow(root$region)
+n_reg <- nrow(root$region)
 
 # Add standard errors 
 source(paste0(path$Subroutines,"/SE_LogRegression.R"))
@@ -52,21 +58,21 @@ for(i in 1:nrow(data))
   # Get root_code of regions 
   reg <- data$Code[i]
 
-  if(reg == 1) reg_range <- paste0("2-",as.character(reg_max))
+  if(reg == 1) reg_range <- paste0("2-",as.character(n_reg))
 
-  if(reg == 2) reg_range <- paste0("[1,3-",as.character(reg_max),"]")
+  if(reg == 2) reg_range <- paste0("[1,3-",as.character(n_reg),"]")
 
-  if(reg == reg_max) reg_range <- paste0("1-",as.character(reg_max-1))
+  if(reg == n_reg) reg_range <- paste0("1-",as.character(n_reg-1))
 
-  if(reg == (reg_max-1)) reg_range <- paste0("[1-",as.character(reg_max-2),",",as.character(reg_max),"]")
+  if(reg == (n_reg-1)) reg_range <- paste0("[1-",as.character(n_reg-2),",",as.character(n_reg),"]")
 
-  if(reg > 2 & reg < (reg_max-1)) {
+  if(reg > 2 & reg < (n_reg-1)) {
     reg_range <- paste0("[1-",as.character(reg-1),",",as.character(reg+1),"-",
-                        as.character(reg_max),"]") }
+                        as.character(n_reg),"]") }
 
   # Read import value
   value <- as.character(data$Quantity[i])
-  # Set SE to 5%
+  # Set SE
   SE <- as.character(data$SE[i])
   reg_name <- root$region$Name[reg]
   reg <- as.character(reg)

Rscripts/datafeeds_code/datafeed_PIOLab_IRPextraction.R

@@ -3,12 +3,6 @@
 datafeed_name <- "IRPextraction"
 print(paste0("datafeed_PIOLab_",datafeed_name," initiated."))
 
-# Create wrapper for writing clean data 
-WriteFile <- function(d,t)
-{
-  write.table(d,file = t,row.names = FALSE,col.names = FALSE,sep = ",",na = "NaN")
-}
-
 ################################################################################
 # Set library path when running on suphys server
 if(Sys.info()[1] == "Linux"){
@@ -19,72 +13,58 @@ if(Sys.info()[1] == "Linux"){
 ################################################################################
 # Initializing R script (load R packages and set paths to folders etc.)
 source(paste0(root_folder,"Rscripts/Subroutines/InitializationR.R"))
-
+
+path["df_Processed"] <- paste0(path$Processed,"/",datafeed_name)
+
+# Load function to write arrays to files
+source(paste0(path$Subroutines,"/Numbers2File.R"))
+
 # Loading raw data
 source(paste0(path$Subroutines,"/Read_ExtractionIRP.R"))
 
 # Loading function for estimating SE with linear regression
 source(paste0(path$Subroutines,"/SE_LogRegression.R"))
 
+# read upper and lower error bounds from settings file
 RSE <- filter(read.xlsx(path$RSE_settings),Item == datafeed_name)
 
+# Estimate standard error
 data <- SE_LogRegression(data,RSE$Minimum,RSE$Maximum)
 
 n_reg <- nrow(root$region) # Number of root regions
 
-data_new <- matrix(NaN,nrow = n_reg,ncol = 2)
+df <- data.frame("RHS" = rep(NaN,n_reg), "SE" = rep(NaN,n_reg))
+df$RHS[data$Code] <- data$Quantity
+df$SE[data$Code] <- data$SE
 
-data_new[data$Code,1] <- data$Quantity
-data_new[data$Code,2] <- data$SE
-data <- as.data.frame(data_new)
-colnames(data) <- c("RHS","SE")
-remove(data_new)
+# Check if folder with processed data exists, in case delete and create empty one
+if(dir.exists(path$df_Processed)) unlink(path$df_Processed,recursive = TRUE) 
+dir.create(path$df_Processed)
 
 # Create empty ALANG table with header
 source(paste0(path$Subroutines,"/makeALANGheadline.R"))
 
-# Check if folder with processed data exists, in case delete and create empty one
-path_set <- paste0(path$root,"ProcessedData/",datafeed_name)
-if(dir.exists(path_set)) unlink(path_set,recursive = TRUE) 
-dir.create(path_set)
-
-# Set path to Sector Concordance
-filename <- list("IndAgg" ="Sector Aggregators/20IndV1_SectorAggregatorIndustries.csv")
-
-IndAgg <- read.csv(paste0(path$Concordance,"/",filename$IndAgg),header = FALSE) 
-IndAgg <- as.matrix(IndAgg)
-
-# Create Root2Root region concordance (identity )
-RegAgg <- diag(length(root$region$Code)) # Create region aggregator
-
-# Set name of concordance
-filename["RegAgg"] <- "Root2Root_Reg_Concordance.csv"
-
-# Write to folder 
-WriteFile(RegAgg,paste0(path$Concordance,"/",filename$RegAgg))
-
-# Define section of path to processed data file
-filename["RHS"] <- paste0(path$Processed,"/",datafeed_name,"/",year,"_")
-
-for(i in root$region$Code) 
+for(i in data$Code) 
 {
-  data_sel <- data.frame("RSE" = rep(NaN,n_reg),"SE" = rep(NaN,n_reg))
+  sel <- df
+  sel[-i,] <- NaN
 
-  data_sel$RSE[i] <- data$RHS[i]
-  data_sel$SE[i] <- data$SE[i]
+  filename <- list("RHS" = paste0("/",datafeed_name,"/",datafeed_name,"_RHS_",year,
+                                  "_",root$region$RootCountryAbbreviation[i],".csv"),
+                   "SE" = paste0("/",datafeed_name,"/",datafeed_name,"_SE_",year,
+                                 "_",root$region$RootCountryAbbreviation[i],".csv"))
 
   # Write RHS and SE data to folder
-  WriteFile(data_sel$RSE,paste0(filename$RHS,"RHS_",root$region$RootCountryAbbreviation[i],"_.csv")) 
-  WriteFile(data_sel$SE[i],paste0(filename$RHS,"SE_",root$region$RootCountryAbbreviation[i],"_.csv")) 
+  Numbers2File(sel$RHS,paste0(path$Processed,filename$RHS)) 
+  Numbers2File(sel$SE,paste0(path$Processed,filename$SE)) 
 
-  # Add command for domestic Use table
-  ALANG <- add_row(ALANG,'#' = i)
+  ALANG <- add_row( ALANG,
+                   '1' = paste0("DataFeed IRP Extraction ",year," ",root$region$Name[i]),
+                   'Value' = paste0("DATAPATH",filename$RHS),
+                   'S.E.' = paste0("DATAPATH",filename$SE) )
 }
 
-ALANG$`1` <- paste0("DataFeed IRP Extraction ",year," ",root$region$Name)
-ALANG$Value <- paste0(filename$RHS,"RHS_",root$region$RootCountryAbbreviation,"_.csv")                   
-ALANG$S.E. <- paste0(filename$RHS,"SE_",root$region$RootCountryAbbreviation,"_.csv")
-
+ALANG$`#` <- 1:nrow(ALANG)
 ALANG$Coef1 <- "1"
 ALANG$Years <- "1"
 ALANG$Margin <- "1"
@@ -108,4 +88,3 @@ ALANG$`Post-Map` <- ""
 source(paste0(path$root,"Rscripts/datafeeds_code/datafeed_subroutines/WriteALANG2Folder.R"))
 
 print(paste0("datafeed_PIOLab_",datafeed_name," finished."))
-