From 4da4637e82677d80988b031ee814fbae0c0c7e2b Mon Sep 17 00:00:00 2001 From: Robin Gerzaguet Date: Mon, 1 Jul 2024 16:20:00 +0200 Subject: [PATCH] Oracle and WiSig 2 --- src/WiSig_Database/src/Struct_data.jl | 96 ++++++++ src/WiSig_Database/src/WiSig_Database.jl | 291 +++++++++++++++++++++++ src/WiSig_Database/src/utils.jl | 126 ++++++++++ 3 files changed, 513 insertions(+) create mode 100644 src/WiSig_Database/src/Struct_data.jl create mode 100644 src/WiSig_Database/src/WiSig_Database.jl create mode 100644 src/WiSig_Database/src/utils.jl diff --git a/src/WiSig_Database/src/Struct_data.jl b/src/WiSig_Database/src/Struct_data.jl new file mode 100644 index 0000000..ecdbf71 --- /dev/null +++ b/src/WiSig_Database/src/Struct_data.jl @@ -0,0 +1,96 @@ + +Base.@kwdef mutable struct Data_WiSig + File_name::String = "DataBases/WiSig/ManySig/pkl_wifi_ManySig/ManySig.pkl" + name::String = "WiSig" + nbTx::Int = 6 + nbSignals::Int = 1000 + Chunksize::Int = 256 + features::String= "IQsamples" + txs = 1:6 + rxs = 1 + days = 1 + equalized= 2 + Normalisation::Bool = true + pourcentTrain::Float64 =0.9 + Augmentation_Value::Data_Augmented = Augmentation.Data_Augmented() +end + + + +function Data_WiSig(; kwargs...) + if haskey(kwargs, :File_name) + File_name = kwargs[:File_name] + else + File_name="DataBases/WiSig/ManySig/pkl_wifi_ManySig/ManySig.pkl" + end + + if haskey(kwargs, :name) + name = kwargs[:name] + else + name="WiSig" + end + + if haskey(kwargs, :nbTx) + nbTx = kwargs[:nbTx] + else + nbTx = 6 + end + + if haskey(kwargs, :nbSignals) + nbSignals = kwargs[:nbSignals] + else + nbSignals = 1000 + end + + if haskey(kwargs, :Chunksize) + Chunksize = kwargs[:Chunksize] + else + Chunksize = 256 + end + + if haskey(kwargs, :features) + features = kwargs[:features] + else + features = "IQsamples" + end + + + if haskey(kwargs, :txs) + txs = kwargs[:txs] + else + txs = 1:6 + end + if haskey(kwargs, :rxs) + rxs = kwargs[:rxs] + else + rxs = 1 + end + if haskey(kwargs, :days) + days = kwargs[:days] + else + days = 1 + end + if haskey(kwargs, :equalized) + equalized = kwargs[:equalized] + else + equalized =2 + end + if haskey(kwargs, :Normalisation) + Normalisation = kwargs[:Normalisation] + else + Normalisation = true + end + if haskey(kwargs, :pourcentTrain) + pourcentTrain = kwargs[:pourcentTrain] + else + pourcentTrain = 0.9 + end + + if haskey(kwargs, :Augmentation_Value) + Augmentation_Value = kwargs[:Augmentation_Value] + else + Augmentation_Value = Data_Augmented() + end + return Data_WiSig(File_name,name, nbTx, nbSignals, Chunksize,features, txs,rxs,days,equalized,Normalisation,pourcentTrain,Augmentation_Value) +end + diff --git a/src/WiSig_Database/src/WiSig_Database.jl b/src/WiSig_Database/src/WiSig_Database.jl new file mode 100644 index 0000000..b9ef067 --- /dev/null +++ b/src/WiSig_Database/src/WiSig_Database.jl @@ -0,0 +1,291 @@ +module WiSig_Database +using Random +using Statistics +using DelimitedFiles +using CSV +using DataFrames +using Infiltrator +include("../../Augmentation/src/Augmentation.jl") +using .Augmentation + +include("utils.jl") + +export setWiSigcsv +export create_X_Y_WiSig +export loadCSV_WiSig +include("Struct_data.jl") +export Data_WiSig + + +""" +Create train and test dataset (here matrixes and not dataloader) for a given distance in feet. If the keyword "all" is used the train and test sets uses all distances. +(X_train,X_test,Y_train,Y_test) = create_X_and_Y(filename,txs,rxs,days,equalized,ChunkSize,pourcentTrain,augmentationType,chanel) +""" +function create_X_Y_WiSig(Param_Data) + nbChunks = Param_Data.nbSignals *Param_Data.nbTx * size(Param_Data.days,1) # size(tupTrain.bigMat,3) + size(tupTest.bigMat,3) + nbTrain = Int(round(Param_Data.pourcentTrain*nbChunks)) + nbTest = nbChunks - nbTrain # size(tupTest.bigMat,3) + + shuffle=true + ChunkSize = Param_Data.Chunksize + # Loading Data and create Matrix for train and test + (X_train,X_test,Y_train,Y_test)= Pickle_to_matrix(Param_Data) + # ---------------------------------------- + # --- Shuffle signals with labels + # ---------------------------------------- + if Param_Data.Augmentation_Value.augmentationType=="augment" + nbAugment = Param_Data.Augmentation_Value.nb_Augment + @info nbAugment + channel =Param_Data.Augmentation_Value.Channel + seed_channel = Param_Data.Augmentation_Value.seed_channel + burstSize = Param_Data.Augmentation_Value.burstSize + N = Int(Param_Data.nbSignals * Param_Data.pourcentTrain) # on génère toujours nbSignaux par canal + ( X_trainTemp,Y_trainTemp)=Augmentation.Add_diff_Channel_train_test(X_train,Y_train,N,channel,Param_Data.Chunksize,Param_Data.Augmentation_Value.nb_Augment,Param_Data.nbTx,seed_channel,burstSize) + @info size(X_train) + # TEST + X_train = X_trainTemp + Y_train = Y_trainTemp + end + + if shuffle + X_trainS = zeros(Float32,ChunkSize,2,nbTrain*Param_Data.Augmentation_Value.nb_Augment) + Y_trainS = zeros(Float32,Param_Data.nbTx,nbTrain*Param_Data.Augmentation_Value.nb_Augment) + X_testS = zeros(Float32,ChunkSize,2,nbTest) + Y_testS = zeros(Float32,Param_Data.nbTx,nbTest) + indexes = randperm(Int(size(X_train,3))) + for i =1 :1 :size(X_train,3) + X_trainS[:,:,(i)] = X_train[:,:,(indexes[i])] + Y_trainS[:,(i)] = Y_train[:,(indexes[i])] + end + indexes = randperm(Int(size(X_test,3))) + for i =1 :1 :size(X_test,3) + X_testS[:,:,(i)] = X_test[:,:,(indexes[i])] + Y_testS[:,(i)] = Y_test[:,(indexes[i])] + end + X_train=X_trainS + Y_train=Y_trainS + X_test=X_testS + Y_test=Y_testS + end + # ---------------------------------------- + # --- Normalized data + # ---------------------------------------- + if Param_Data.Normalisation + (moy,std_val) = preProcessing!(X_train,nothing,nothing) + (moy,std_val) = preProcessing!(X_test,moy,std_val) + end + if Param_Data.features == "Module_angle" + X_train_Mod = zeros(Float32,ChunkSize,2,nbTrain)#*nb_Augment) + X_test_Mod = zeros(Float32,ChunkSize,2,nbTest)#*nbAugment_Test) + X_train_Mod[:,1,:] .= abs2.(X_train[:,1,:]+ im *X_train[:,2,:]) + X_train_Mod[:,2,:] .= angle.(X_train[:,1,:]+ im *X_train[:,2,:]) + X_train = X_train_Mod + X_test = X_test_Mod + end + return (X_train,Y_train,X_test,Y_test) +end + + + +""" +Apply normalisation to input data +""" +function preProcessing!(X,moy,std_val) + if isnothing(moy) + moy_reel = mean(X[:,1,:]) + moy_ima = mean(X[:,2,:]) + else + moy_reel = real(moy) + moy_ima = imag(moy) + end + if isnothing(std_val) + std_val_reel = std(X[:,1,:]) + std_val_ima = std(X[:,2,:]) + else + std_val_reel = real(std_val) + std_val_ima = imag(std_val) + end + X[:,1,:] .= (X[:,1,:] .- moy_reel)./std_val_reel + X[:,2,:] .= (X[:,2,:] .- moy_ima )./std_val_ima + return (moy_reel+1im*moy_ima,std_val_reel+1im*std_val_ima) +end + + +""" Transforme la matrice des labels en un vecteur d'indice 0-(NbRadios-1) """ +function create_bigMat_Labels_Tx(new_bigLabels) + bigLabels = zeros(Int,size(new_bigLabels)[2]) + for i in 1:size(new_bigLabels)[2] + for j in 1:size(new_bigLabels)[1] + if new_bigLabels[j,i] == 1 + bigLabels[i] = j-1; + end + end + end + + return bigLabels +end + +#= +""" Function that load data from the CSV file for WiSig database +""" +function loadCSV_WiSig(Param_Data) + ChunkSize = Param_Data.Chunksize + suffix = "$(Param_Data.txs)_$(Param_Data.rxs)_$(Param_Data.days)_$(Param_Data.equalized)_$(Param_Data.pourcentTrain)_$(ChunkSize)" + X_train = CSV.read("CSV_Files/WiSig/$(suffix)/bigMatTrain_$suffix.csv", DataFrame) |> Matrix .|> Float32 |> transpose |> collect + Y_train = CSV.read("CSV_Files/WiSig/$(suffix)/bigLabelsTrain_$suffix.csv", DataFrame) |> Matrix .|> Float32 |> transpose |> collect + X_test = CSV.read("CSV_Files/WiSig/$(suffix)/bigMatTest_$suffix.csv", DataFrame) |> Matrix |> transpose |> collect + Y_test = CSV.read("CSV_Files/WiSig/$(suffix)/bigLabelsTest_$suffix.csv", DataFrame) |> Matrix |> transpose |> collect + @show Y_test[1] + @show size(Y_test) + @show nbRadios = 1 + Int(maximum(Y_test)) # Radio index is between 0 and nbRadio -1 + + ChunkSize = size(X_train,1) ÷ 2 + X_train = reshape(X_train,ChunkSize,2,:)[:,:,1:end÷2] + X_test = reshape(X_test ,ChunkSize,2,:)[:,:,1:end÷2] + Y_train = transformLabels(Y_train,nbRadios)[:,1:end÷2] + Y_test = transformLabels(Y_test,nbRadios)[:,1:end÷2] + @show size(X_train),size(X_test),size(Y_train),size(Y_test) + return (X_train,Y_train,X_test,Y_test) +end +=# + +""" Function that load data from the CSV file for Synthetic database +""" +function loadCSV_WiSig(Param_Data) + + nbChunks=Int(Param_Data.nbTx*Param_Data.nbSignals * size(Param_Data.days,1)) + nbTrain = Int(round(Param_Data.pourcentTrain*nbChunks)) + nbTest = nbChunks - nbTrain + #if augmentationType == "No_channel" + suffix = "$(Param_Data.txs)_$(Param_Data.rxs)_$(Param_Data.days)_$(Param_Data.equalized)_$(Param_Data.pourcentTrain)_$(Param_Data.Chunksize)" + savepath = "./CSV_Files/WiSig/$(suffix)" + #=else + channel = Param_Data.Augmentation_Value.Channel + channel_Test = Param_Data.Augmentation_Value.Channel_Test + nbAugment = Param_Data.Augmentation_Value.nb_Augment + savepath = "./CSV_Files/$(Param_Data.Augmentation_Value.augmentationType)_$(Param_Data.nbTx)_$(Param_Data.Chunksize)/$(Param_Data.E)_$(Param_Data.S)/$(Param_Data.E)_$(Param_Data.S)_$(Param_Data.C)_$(Param_Data.RFF)_$(Param_Data.nbSignals)_$(Param_Data.nameModel)_$(Param_Data.Augmentation_Value.Channel)_$(Param_Data.Augmentation_Value.Channel_Test)_nbAugment_$(Param_Data.Augmentation_Value.nb_Augment)" + nbTrain = nbTrain * nbAugment + if augmentationType == "1channelTest" + nbTest = nbTest * 1 + elseif augmentationType == "same_channel" + nbTest = nbTest * 1 + else augmentationType == "augment" + nbTest = nbTest * 100 + end + end + =# + # Labels + fileLabelTest= "$(savepath)/bigLabelsTest_$suffix.csv" + Y_testTemp = Matrix(DataFrame(CSV.File(fileLabelTest;types=Int64,header=false))) + fileLabelTrain= "$(savepath)/bigLabelsTrain_$suffix.csv" + Y_trainTemp = Matrix(DataFrame(CSV.File(fileLabelTrain;types=Int64,header=false))) + # Data + fileDataTest= "$(savepath)/bigMatTest_$suffix.csv" + X_testTemp = Matrix(DataFrame(CSV.File(fileDataTest;types=Float32,header=false))) + fileDataTrain= "$(savepath)/bigMatTrain_$suffix.csv" + X_trainTemp = Matrix(DataFrame(CSV.File(fileDataTrain;types=Float32,header=false))) + X_train = zeros(Float32, Param_Data.Chunksize,2,nbTrain*Param_Data.Augmentation_Value.nb_Augment) + X_test = zeros(Float32, Param_Data.Chunksize,2,nbTest) + Y_train = zeros(Param_Data.nbTx,nbTrain*Param_Data.Augmentation_Value.nb_Augment) + Y_test = zeros(Param_Data.nbTx,nbTest) + + for i in 1:size(X_trainTemp)[1] + X_train[:,1,i]=X_trainTemp[i,1:Param_Data.Chunksize] + X_train[:,2,i]=X_trainTemp[i,Param_Data.Chunksize+1:Param_Data.Chunksize+Param_Data.Chunksize] + end + for i in 1:size(X_testTemp)[1] + X_test[:,1,i]=X_testTemp[i,1:Param_Data.Chunksize] + X_test[:,2,i]=X_testTemp[i,Param_Data.Chunksize+1:Param_Data.Chunksize+Param_Data.Chunksize] + end + for i in 1:size(Y_trainTemp)[1] + Y_train[Y_trainTemp[i]+1,i]=1 + end + for i in 1:size(Y_testTemp)[1] + Y_test[Y_testTemp[i]+1,i]=1 + end + return (X_train,Y_train,X_test,Y_test) +end + + + + + + + +""" +Transform a vector of label (with each index a number of radio) into a matrix of 0 with a 1 per column associated to the radio index +""" +function transformLabels(Y,nbRadios) + Z = zeros(Int,nbRadios,length(Y)) + for n in eachindex(Y) + Z[1+Int(Y[n]),n] = 1 + end + return Z +end + + + +""" Function that create 4 csv files based on pkl file(s) +CSV files: +- Training Data +- Test Data +- Training Labels +- Test Labels """ +function setWiSigcsv(Param_Data) + + + (bigMatTrain,bigLabels_Train,bigMatTest,bigLabels_Test) = create_X_Y_WiSig(Param_Data) + + ################# + #= + Param_Data.nbSignals=10000 + Param_Data.pourcentTrain = 0.99 + snr=10 + BigMatTrain_New=zeros(Param_Data.Chunksize,2,Int(Param_Data.nbTx*Param_Data.nbSignals*Param_Data.pourcentTrain)) + BigLabelsTrain_New=zeros(Param_Data.nbTx,Int(Param_Data.nbSignals*Param_Data.pourcentTrain*Param_Data.nbTx)) + for i = 1 :1 : size(bigMatTrain)[3] + x=bigMatTrain[:,:,i] + powSig = sum(abs.(x).^2) / length(x); + # --- Evaluation of noise power + powNoise = sqrt(powSig) * 10^(-snr/20); + # --- Create a complex random sequence as additive noise + for k=1:1:11 + n = randn(length(x))* powNoise; + BigMatTrain_New[:,1,(i-1)*11+k] = x[:,1] + n[1:256] + BigMatTrain_New[:,2,(i-1)*11+k] = x[:,2] + n[257:512] + BigLabelsTrain_New[:,(i-1)*11+k] = bigLabels_Train[:,i] + end + end + bigLabels_Train =BigLabelsTrain_New + bigMatTrain= BigMatTrain_New +=# + ###############@ + bigLabels_Train = create_bigMat_Labels_Tx(bigLabels_Train) + bigLabels_Test = create_bigMat_Labels_Tx(bigLabels_Test) + savepath = "CSV_Files/WiSig/$(Param_Data.txs)_$(Param_Data.rxs)_$(Param_Data.days)_$(Param_Data.equalized)_$(Param_Data.pourcentTrain)_$(Param_Data.Chunksize)" + !ispath(savepath) && mkpath(savepath) + open("$(savepath)/bigMatTrain_$(Param_Data.txs)_$(Param_Data.rxs)_$(Param_Data.days)_$(Param_Data.equalized)_$(Param_Data.pourcentTrain)_$(Param_Data.Chunksize).csv","w") do io + for i in 1:size(bigMatTrain)[3] + writedlm(io,[vcat(bigMatTrain[:,:,i][:,1],bigMatTrain[:,:,i][:,2])]) + end + end + + open("$(savepath)/bigMatTest_$(Param_Data.txs)_$(Param_Data.rxs)_$(Param_Data.days)_$(Param_Data.equalized)_$(Param_Data.pourcentTrain)_$(Param_Data.Chunksize).csv","w") do io + for i in 1:size(bigMatTest)[3] + writedlm(io,[vcat(bigMatTest[:,:,i][:,1],bigMatTest[:,:,i][:,2])]) + end + end + open("$(savepath)/bigLabelsTrain_$(Param_Data.txs)_$(Param_Data.rxs)_$(Param_Data.days)_$(Param_Data.equalized)_$(Param_Data.pourcentTrain)_$(Param_Data.Chunksize).csv","w") do io + for i in 1:size(bigLabels_Train)[1] + writedlm(io,[bigLabels_Train[i]]) + end + end + open("$(savepath)/bigLabelsTest_$(Param_Data.txs)_$(Param_Data.rxs)_$(Param_Data.days)_$(Param_Data.equalized)_$(Param_Data.pourcentTrain)_$(Param_Data.Chunksize).csv","w") do io + for i in 1:size(bigLabels_Test)[1] + writedlm(io,[bigLabels_Test[i]]) + end + end +end + + +end \ No newline at end of file diff --git a/src/WiSig_Database/src/utils.jl b/src/WiSig_Database/src/utils.jl new file mode 100644 index 0000000..43ebb10 --- /dev/null +++ b/src/WiSig_Database/src/utils.jl @@ -0,0 +1,126 @@ +using Pickle + + +export Pickle_to_matrix +"""Creer une matrice bigMat pour des Txs, Rxs, jours, equalized or not donnés \n +# Parametres sous la forme de liste 1:6, 1:12; 1:4, 1:2 et 1 ou 2 || [,] +""" +function Pickle_to_matrix(Param_Data) + # --- Load file + + obj = myunpickle(Param_Data.File_name) + # --- Estimate the number of burst for train and test + nbBurstTotal = evaluate_total_number_of_burst_Spec(obj,Param_Data.txs,Param_Data.rxs,Param_Data.days,Param_Data.equalized) + nbBurstTotalTrain = Int(nbBurstTotal*Param_Data.pourcentTrain) + nbBurstTotalTest = nbBurstTotal - nbBurstTotalTrain + # --- Create and init temporary matrix + bigMatTemp = zeros(Float32,Param_Data.Chunksize,2,nbBurstTotal) + bigLabelsTemp = zeros(Int,Param_Data.nbTx,nbBurstTotal) + # --- Create and init Data and labels matrix + bigMatTrain = zeros(Float32,Param_Data.Chunksize,2,nbBurstTotalTrain) + bigMatTest = zeros(Float32,Param_Data.Chunksize,2,nbBurstTotalTest) + bigLabelsTrain = zeros(Int,Param_Data.nbTx,nbBurstTotalTrain) + bigLabelsTest = zeros(Int,Param_Data.nbTx,nbBurstTotalTest) + tmp_somme = 0 + tmp_sommeTrain = 0 + tmp_sommeTest = 0 + # --- browse all of the options + for tx in Param_Data.txs + for rx in Param_Data.rxs + for day in Param_Data.days + for eq in Param_Data.equalized + # -- Shared each groups of signal between Train and Test + # --- Load Data and determine number of burst () + data::Array{Float32} = obj["data"][tx][rx][day][eq] + (nbBurst,_,_) = size(data) + nbBurstTrain = Int( nbBurst * Param_Data.pourcentTrain) + nbBurstTest = nbBurst - nbBurstTrain + # ici il faut diviser on aura nbBurst à mult par pourcentrain pour savoir combien on prend de Burst en train pour cette config (TX, rx,jours) + if nbBurst != 0 + # On utilise la fonction create_bigMat_ManySig pour créer des vecteurs temporaire qu'on vient diviser pour Train et Test + (bigMatTemp[:,:,tmp_somme+1:tmp_somme+nbBurst],bigLabelsTemp[:,tmp_somme+1:tmp_somme+nbBurst]) = create_bigMat_ManySig(data,tx,Param_Data.nbTx,Param_Data.Chunksize) # On sait que 1000 samples + bigMatTrain[:,:,tmp_sommeTrain+1:tmp_sommeTrain+nbBurstTrain] = bigMatTemp[:,:,tmp_somme+1:tmp_somme+nbBurstTrain]#On sait que 1000 samples + bigLabelsTrain[:,tmp_sommeTrain+1:tmp_sommeTrain+nbBurstTrain] = bigLabelsTemp[:,tmp_somme+1:tmp_somme+nbBurstTrain] + bigMatTest[:,:,tmp_sommeTest+1:tmp_sommeTest+nbBurstTest] = bigMatTemp[:,:,tmp_somme+nbBurstTrain+1:tmp_somme+nbBurstTrain+nbBurstTest] + bigLabelsTest[:,tmp_sommeTest+1:tmp_sommeTest+nbBurstTest] = bigLabelsTemp[:,tmp_somme+nbBurstTrain+1:tmp_somme+nbBurstTrain+nbBurstTest] #On sait que 1000 samples + tmp_somme +=nbBurst + tmp_sommeTrain +=nbBurstTrain + tmp_sommeTest +=nbBurstTest + end + end + end + end + end + bm::Array{Float32} = convert(Array{Float32},bigMatTrain) + bt::Array{Float32} = convert(Array{Float32},bigMatTest) + lm::Array{Int} = convert(Array{Int},bigLabelsTrain) + lt::Array{Int} = convert(Array{Int},bigLabelsTest) + + return(bm,bt,lm,lt) +end + + +""" +Evalue la taille pour allocation // parametre sous la forme de liste 1:6, 1:12; 1:4, 1:2 et 1 ou 2 +""" +function evaluate_total_number_of_burst_Spec(obj,txs,rxs,days,equalized) + nbBurst = 0 + for tx ∈ txs + for rx ∈ rxs + for d ∈ days + for eq ∈ equalized + data = obj["data"][tx][rx][d][eq] + (tmp,_,_) = size(data) + nbBurst += tmp + end + end + end + end + return nbBurst +end + +"""Creer une matrice bigMat pour un Tx, un Rx, un jour, equalized or not données""" +function create_bigMat_ManySig(data,tx,nbRadios,ChunkSize) + cnt = 0 + (nbBurst,_,_) = size(data) + bigMat = zeros(Float32,ChunkSize,2,nbBurst) + bigLabels = zeros(Int,nbRadios,nbBurst) + if nbBurst != 0 + # --- Recreate data matrix + theView = @views bigMat[:,:,cnt .+ (1:nbBurst)] + reshapeData!(theView,data) + bigLabels[tx , cnt .+ (1:nbBurst)] .= 1 + # --- Update counter + cnt += nbBurst + end + return (bigMat,bigLabels) +end + +""" Transform Wisig data (nbBurst x ChunkSize x 2) into Oracle compatible format which is ChunkSize x 2 x nbBurst +""" +function reshapeData!(y,x::Array{Float32}) + nbBurst = size(x,1) + @assert size(x) == (nbBurst,256,2) "Size of x is incorrect" + @assert size(y) == (256,2,nbBurst) "Size of y is incorrect" + for n in 1 : nbBurst + y[:,:,n] = Float32.(x[n,:,:]) + end +end + +"""Just to check what the .pkl file is """ +function myunpickle(filename) + open(filename,"r") do io + Pickle.npyload(io) + end +end + +"""Permet de concatener en le signal I + iQ """ +function concaten!(tmp_concat,indexList,bigMat) + j = 1 + for i in indexList + tmp_concat[(j-1)*256+1:j*256] = bigMat[:,:,i][:,1] + bigMat[:,:,i][:,2]im + j=j+1 + end +end + +