Introduce data explicit scoping to OMP directives

PaulMullowney · Jan 29, 2025 · 9b19273 · 9b19273
1 parent 1000cbd
commit 9b19273
Show file tree

Hide file tree

Showing 9 changed files with 88 additions and 55 deletions.
diff --git a/src/trans/gpu/internal/fsc_mod.F90 b/src/trans/gpu/internal/fsc_mod.F90
@@ -101,6 +101,10 @@ SUBROUTINE FSC(ALLOCATOR,HFSC,PREEL_COMPLEX, KF_FS, KF_UV, KF_SCALARS, KUV_OFFSE
   IINC=-1
 ENDIF
 
+#ifdef OMPGPU
+!$OMP TARGET DATA &
+!$OMP& MAP(PRESENT,ALLOC:D,D_NPTRLS,D_NSTAGTF,PREEL_COMPLEX,F,F_RACTHE,G,G_NMEN,G_NLOEN,R,R_NSMAX)
+#endif
 #ifdef ACCGPU
 !$ACC DATA &
 !$ACC& PRESENT(D,D_NPTRLS,D_NSTAGTF,PREEL_COMPLEX,F,F_RACTHE,G,G_NMEN,G_NLOEN,R,R_NSMAX)
@@ -116,9 +120,9 @@ SUBROUTINE FSC(ALLOCATOR,HFSC,PREEL_COMPLEX, KF_FS, KF_UV, KF_SCALARS, KUV_OFFSE
 !*       1.1      U AND V.
 #ifdef OMPGPU
 !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
-!$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_UV,ZACHTE2) &
-!$OMP& SHARED(IBEG,IEND,IINC,KF_UV,R,OFFSET_VAR,G,D,KF_FS,KUV_OFFSET,F,PREEL_COMPLEX) &
-!$OMP& MAP(TO:IBEG,IEND,IINC,KF_UV,OFFSET_VAR,KF_FS,KUV_OFFSET,ZACHTE2)
+!$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_UV,ZACHTE2,JM,JF,KGL) &
+!$OMP& FIRSTPRIVATE(IBEG,IEND,IINC,OFFSET_VAR,KF_UV,KUV_OFFSET,KF_FS) &
+!$OMP& SHARED(R,G,D,F,PREEL_COMPLEX) NOWAIT
 #endif
 #ifdef ACCGPU
 !$ACC PARALLEL LOOP COLLAPSE(3) DEFAULT(NONE) &
@@ -149,10 +153,9 @@ SUBROUTINE FSC(ALLOCATOR,HFSC,PREEL_COMPLEX, KF_FS, KF_UV, KF_SCALARS, KUV_OFFSE
 IF (KSCALARS_NSDER_OFFSET >= 0) THEN
 #ifdef OMPGPU
   !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
-  !$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_KSCALARS_NSDER,ZACHTE2) &
-  !$OMP& SHARED(IBEG,IEND,IINC,KF_SCALARS,R,OFFSET_VAR,G,D,KF_FS,KSCALARS_NSDER_OFFSET,F,&
-  !$OMP&        PREEL_COMPLEX) &
-  !$OMP& MAP(TO:IBEG,IEND,IINC,KF_SCALARS,OFFSET_VAR,KF_FS,KSCALARS_NSDER_OFFSET)
+  !$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_KSCALARS_NSDER,ZACHTE2,KGL,JF,JM) &
+  !$OMP& FIRSTPRIVATE(IBEG,IEND,IINC,OFFSET_VAR,KF_SCALARS,KSCALARS_NSDER_OFFSET,KF_FS) &
+  !$OMP& SHARED(R,G,D,F,PREEL_COMPLEX) NOWAIT
 #endif
 #ifdef ACCGPU
   !$ACC PARALLEL LOOP COLLAPSE(3) DEFAULT(NONE) PRIVATE(IGLG,IOFF_LAT,IOFF_KSCALARS_NSDER,ZACHTE2,KGL,JF,JM) &
@@ -189,10 +192,9 @@ SUBROUTINE FSC(ALLOCATOR,HFSC,PREEL_COMPLEX, KF_FS, KF_UV, KF_SCALARS, KUV_OFFSE
 IF (KUV_EWDER_OFFSET >= 0) THEN
 #ifdef OMPGPU
   !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
-  !$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_UV,IOFF_UV_EWDER,RET_REAL,RET_COMPLEX,ZACHTE2) &
-  !$OMP& SHARED(IBEG,IEND,IINC,KF_UV,ILOEN_MAX,OFFSET_VAR,G,D,KF_FS,KUV_OFFSET,KUV_EWDER_OFFSET,F,&
-  !$OMP&        PREEL_COMPLEX) &
-  !$OMP& MAP(TO:IBEG,IEND,IINC,KF_UV,ILOEN_MAX,OFFSET_VAR,KF_FS,KUV_OFFSET,KUV_EWDER_OFFSET)
+  !$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_UV,IOFF_UV_EWDER,RET_REAL,RET_COMPLEX,ZACHTE2,JM,JF,KGL) &
+  !$OMP& FIRSTPRIVATE(IBEG,IEND,IINC,OFFSET_VAR,KF_UV,KUV_EWDER_OFFSET,KUV_OFFSET,KF_FS,ILOEN_MAX) &
+  !$OMP& SHARED(G,D,F,PREEL_COMPLEX) NOWAIT
 #endif
 #ifdef ACCGPU
   !$ACC PARALLEL LOOP COLLAPSE(3) DEFAULT(NONE) PRIVATE(IGLG,IOFF_LAT,IOFF_UV,IOFF_UV_EWDER,RET_REAL,RET_COMPLEX,ZACHTE2,JM,JF,KGL) &
@@ -234,11 +236,9 @@ SUBROUTINE FSC(ALLOCATOR,HFSC,PREEL_COMPLEX, KF_FS, KF_UV, KF_SCALARS, KUV_OFFSE
 IF (KSCALARS_EWDER_OFFSET > 0) THEN
 #ifdef OMPGPU
   !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
-  !$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_SCALARS_EWDER,IOFF_SCALARS,RET_REAL,RET_COMPLEX,ZACHTE2) &
-  !$OMP& SHARED(IBEG,IEND,IINC,KF_SCALARS,ILOEN_MAX,OFFSET_VAR,G,D,KF_FS,KSCALARS_EWDER_OFFSET,&
-  !$OMP&        KSCALARS_OFFSET,F,PREEL_COMPLEX) &
-  !$OMP& MAP(TO:IBEG,IEND,IINC,KF_SCALARS,ILOEN_MAX,OFFSET_VAR,KF_FS,KSCALARS_EWDER_OFFSET,&
-  !$OMP&     KSCALARS_OFFSET)
+  !$OMP& PRIVATE(IGLG,IOFF_LAT,IOFF_SCALARS_EWDER,IOFF_SCALARS,ZACHTE2,RET_REAL,RET_COMPLEX) &
+  !$OMP& FIRSTPRIVATE(IBEG,IEND,IINC,KF_SCALARS,OFFSET_VAR,KSCALARS_EWDER_OFFSET,KSCALARS_OFFSET,KF_FS,ILOEN_MAX) &
+  !$OMP& SHARED(G,D,F,PREEL_COMPLEX) NOWAIT
 #endif
 #ifdef ACCGPU
   !$ACC PARALLEL LOOP COLLAPSE(3) DEFAULT(NONE) PRIVATE(IGLG,IOFF_LAT,IOFF_SCALARS_EWDER,IOFF_SCALARS,ZACHTE2,RET_REAL,RET_COMPLEX) &
@@ -281,6 +281,11 @@ SUBROUTINE FSC(ALLOCATOR,HFSC,PREEL_COMPLEX, KF_FS, KF_UV, KF_SCALARS, KUV_OFFSE
 
 !$ACC END DATA
 #endif
+#ifdef OMPGPU
+!$OMP TASKWAIT
+
+!$OMP END TARGET DATA
+#endif
 !     ------------------------------------------------------------------
 END ASSOCIATE
 

diff --git a/src/trans/gpu/internal/leinv_mod.F90 b/src/trans/gpu/internal/leinv_mod.F90
@@ -181,14 +181,14 @@ SUBROUTINE LEINV(ALLOCATOR,PIA,ZINP,ZINP0,ZOUTS,ZOUTA,ZOUTS0,ZOUTA0,KF_LEG)
     !       PIA_2=2+1+(1..4-1)*2 ...3+(0..3)*2 .... 3,5,7,9
 
 #ifdef OMPGPU
-    !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(KM,IA,J) &
+    !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) DEFAULT(NONE) &
+    !$OMP& PRIVATE(KM,IA,J) &
     !$OMP& FIRSTPRIVATE(KF_LEG,IIN_STRIDES0,IIN0_STRIDES0) &
+    !$OMP& SHARED(D,R,ZINP,ZINP0,PIA) &
 #ifdef _CRAYFTN
     !$OMP&
 #else
-    !!TODO: Think about async for OMP
-    !!$OMP& NOWAIT
-    !$OMP&
+    !$OMP& NOWAIT
 #endif
 #endif
 #ifdef ACCGPU
@@ -344,12 +344,12 @@ SUBROUTINE LEINV(ALLOCATOR,PIA,ZINP,ZINP0,ZOUTS,ZOUTA,ZOUTS0,ZOUTA0,KF_LEG)
     !       PIA_2=1+1+(1..5-1)*2 ...2+(0..4)*2 .... 2,4,6,8,10
 
 #ifdef OMPGPU
-    !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(KM,IS,J) &
+    !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) DEFAULT(NONE) &
+    !$OMP& PRIVATE(KM,IS,J) &
     !$OMP& FIRSTPRIVATE(KF_LEG,IIN_STRIDES0,IIN0_STRIDES0) &
+    !$OMP& SHARED(D,R,ZINP,ZINP0,PIA) &
 #ifndef _CRAYFTN
-    !!TODO: Think about async for OMP
-    !!$OMP& NOWAIT
-    !$OMP&
+    !$OMP& NOWAIT
 #else
     !$OMP&
 #endif

diff --git a/src/trans/gpu/internal/ltinv_mod.F90 b/src/trans/gpu/internal/ltinv_mod.F90
@@ -386,6 +386,7 @@ SUBROUTINE LTINV(ALLOCATOR,HLTINV,KF_UV,KF_SCALARS,&
     ENDIF
 
 #ifdef OMPGPU
+    !$OMP TASKWAIT
     !$OMP END TARGET DATA
     !$OMP END TARGET DATA
     !$OMP END TARGET DATA

diff --git a/src/trans/gpu/internal/prfi1b_mod.F90 b/src/trans/gpu/internal/prfi1b_mod.F90
@@ -96,12 +96,12 @@ SUBROUTINE PRFI1B(PIA,PSPEC,KFIELDS,KDIM,KFLDPTR)
     !loop over wavenumber
 
 #ifdef OMPGPU
-   !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) PRIVATE(KM,IASM0,INM) &
+   !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
+   !$OMP& PRIVATE(KM,IASM0,INM) &
    !$OMP& FIRSTPRIVATE(KFIELDS,KDIM) &
+   !$OMP& SHARED(PIA,PSPEC,R,D) &
 #ifndef _CRAYFTN
-   !!TODO: Think about async for OMP
-   !!$OMP& NOWAIT
-   !$OMP&
+   !$OMP& NOWAIT
 #else
    !$OMP&
 #endif

diff --git a/src/trans/gpu/internal/spnsde_mod.F90 b/src/trans/gpu/internal/spnsde_mod.F90
@@ -103,7 +103,14 @@ SUBROUTINE SPNSDE(KF_SCALARS,PEPSNM,PF,PNSD)
 
 #ifdef OMPGPU
 !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
-!$OMP& PRIVATE(KM,IR,II,JI) MAP(TO:KF_SCALARS) SHARED(D,R,PEPSNM,PF,PNSD,KF_SCALARS)
+!$OMP& PRIVATE(KM,IR,II,JI) &
+!$OMP& FIRSTPRIVATE(KF_SCALARS) &
+!$OMP& SHARED(D,R,PEPSNM,PF,PNSD) &
+#ifndef _CRAYFTN
+!$OMP& NOWAIT
+#else
+!$OMP&
+#endif
 #endif
 #ifdef ACCGPU
 !$ACC PARALLEL LOOP DEFAULT(NONE) COLLAPSE(3) PRIVATE(KM,IR,II,JI) FIRSTPRIVATE(KF_SCALARS) &

diff --git a/src/trans/gpu/internal/trltog_mod.F90 b/src/trans/gpu/internal/trltog_mod.F90
@@ -569,6 +569,7 @@ SUBROUTINE TRLTOG(ALLOCATOR,HTRLTOG,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
       ENDDO
 
 #ifdef OMPGPU
+      ! TODO: Make this asynchronous
       !$OMP TARGET DATA MAP(TO:IFLDA(1:IFLDS))
 #endif
 #ifdef ACCGPU
@@ -582,10 +583,12 @@ SUBROUTINE TRLTOG(ALLOCATOR,HTRLTOG,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
       IIN_TO_SEND_BUFR_V = IIN_TO_SEND_BUFR_OFFSET(MYPROC)
       IF (PRESENT(PGP)) THEN
 #ifdef OMPGPU
-        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) &
+        ! TODO: Make this asynchronous
+        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) DEFAULT(NONE) &
         !$OMP&       PRIVATE(JK,JBLK,IFLD,IPOS) &
         !$OMP&       FIRSTPRIVATE(KF_FS,IRECV_WSET_SIZE_V,IRECV_WSET_OFFSET_V, &
-        !$OMP&       IIN_TO_SEND_BUFR_V,NPROMA)
+        !$OMP&       IIN_TO_SEND_BUFR_V,NPROMA) &
+        !$OMP&       SHARED(IFLDA,IIN_TO_SEND_BUFR,PGP,PREEL_REAL)
 #endif
 #ifdef ACCGPU
         !$ACC PARALLEL LOOP COLLAPSE(2) DEFAULT(NONE) PRIVATE(JK,JBLK,IFLD,IPOS) &
@@ -604,10 +607,13 @@ SUBROUTINE TRLTOG(ALLOCATOR,HTRLTOG,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
         ENDDO
       ELSE
 #ifdef OMPGPU
-        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) &
+        ! TODO: Make this asynchronous
+        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) DEFAULT(NONE) &
         !$OMP&       PRIVATE(JK,JBLK,IFLD,IPOS) &
         !$OMP&       FIRSTPRIVATE(KF_FS,IRECV_WSET_SIZE_V,IRECV_WSET_OFFSET_V, &
-        !$OMP&       IIN_TO_SEND_BUFR_V,NPROMA)
+        !$OMP&       IIN_TO_SEND_BUFR_V,NPROMA) &
+        !$OMP&       SHARED(IFLDA,IIN_TO_SEND_BUFR,IGP_OFFSETS,PGPUV,PGP2,PGP3A, &
+        !$OMP&       PGP3B,PREEL_REAL)
 #endif
 #ifdef ACCGPU
         !$ACC PARALLEL LOOP COLLAPSE(2) DEFAULT(NONE) PRIVATE(JK,JBLK,IFLD,IPOS) &
@@ -696,8 +702,11 @@ SUBROUTINE TRLTOG(ALLOCATOR,HTRLTOG,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
       IIN_TO_SEND_BUFR_V = IIN_TO_SEND_BUFR_OFFSET(IPROC)
       ICOMBUFS_OFFSET_V = ICOMBUFS_OFFSET(INS)
 #ifdef OMPGPU
-      !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(IPOS) &
-      !$OMP& FIRSTPRIVATE(KF_FS,ILEN,IIN_TO_SEND_BUFR_V,ICOMBUFS_OFFSET_V)
+      ! TODO: Make this asynchronous
+      !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) DEFAULT(NONE) &
+      !$OMP& PRIVATE(IPOS) &
+      !$OMP& FIRSTPRIVATE(KF_FS,ILEN,IIN_TO_SEND_BUFR_V,ICOMBUFS_OFFSET_V) &
+      !$OMP& SHARED(IIN_TO_SEND_BUFR,ZCOMBUFS,PREEL_REAL)
 #endif
 #ifdef ACCGPU
       !$ACC PARALLEL LOOP DEFAULT(NONE) PRIVATE(IPOS) FIRSTPRIVATE(KF_FS,ILEN,IIN_TO_SEND_BUFR_V, &
@@ -866,10 +875,12 @@ SUBROUTINE TRLTOG(ALLOCATOR,HTRLTOG,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
       IRECV_WSET_SIZE_V = IRECV_WSET_SIZE(ISETW)
       IF (PRESENT(PGP)) THEN
 #ifdef OMPGPU
-        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) &
+        ! TODO: Make this asynchronous
+        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) DEFAULT(NONE) &
         !$OMP&       PRIVATE(JK,JBLK,IFLD,JI) &
         !$OMP&       FIRSTPRIVATE(IRECV_FIELD_COUNT_V,IRECV_WSET_SIZE_V, &
-        !$OMP&       IRECV_WSET_OFFSET_V,NPROMA,ICOMBUFR_OFFSET_V)
+        !$OMP&       IRECV_WSET_OFFSET_V,NPROMA,ICOMBUFR_OFFSET_V) &
+        !$OMP&       SHARED(IFLDA,PGP,ZCOMBUFR)
 #endif
 #ifdef ACCGPU
         !$ACC PARALLEL LOOP COLLAPSE(2) DEFAULT(NONE) PRIVATE(JK,JBLK,IFLD,JI) &
@@ -887,10 +898,12 @@ SUBROUTINE TRLTOG(ALLOCATOR,HTRLTOG,PREEL_REAL,KF_FS,KF_GP,KF_UV_G,KF_SCALARS_G,
         ENDDO
       ELSE
 #ifdef OMPGPU
-        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) &
+        ! TODO: Make this asynchronous
+        !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) DEFAULT(NONE) &
         !$OMP&       PRIVATE(JK,JBLK,IFLD,JI) &
         !$OMP&       FIRSTPRIVATE(IRECV_FIELD_COUNT_V,IRECV_WSET_SIZE_V, &
-        !$OMP&       IRECV_WSET_OFFSET_V,NPROMA,ICOMBUFR_OFFSET_V)
+        !$OMP&       IRECV_WSET_OFFSET_V,NPROMA,ICOMBUFR_OFFSET_V) &
+        !$OMP&       SHARED(IFLDA,IGP_OFFSETS,PGPUV,PGP2,PGP3A,PGP3B,ZCOMBUFR)
 #endif
 #ifdef ACCGPU
         !$ACC PARALLEL LOOP COLLAPSE(2) DEFAULT(NONE) PRIVATE(JK,JBLK,IFLD,JI) &

diff --git a/src/trans/gpu/internal/trmtol_mod.F90 b/src/trans/gpu/internal/trmtol_mod.F90
@@ -160,7 +160,15 @@ SUBROUTINE TRMTOL(ALLOCATOR,HTRMTOL,PFBUF_IN,PFBUF,KF_LEG)
           FROM_RECV = IOFFR(IRANK) + 1
           TO_RECV = FROM_RECV + ILENR(IRANK) - 1
 #ifdef OMPGPU
-          !$OMP TARGET TEAMS MAP(PRESENT,ALLOC:PFBUF,PFBUF_IN) MAP(TO:FROM_RECV,TO_RECV,FROM_SEND,TO_SEND)
+          !$OMP TARGET TEAMS DEFAULT(NONE) &
+          !$OMP& SHARED(PFBUF,PFBUF_IN,FROM_RECV,TO_RECV,FROM_SEND,TO_SEND) &
+          !$OMP& MAP(TO:FROM_RECV,TO_RECV,FROM_SEND,TO_SEND) &
+#ifdef __HIP_PLATFORM_AMD__
+          !$OMP&
+#else
+          ! TODO: Test this
+          !$OMP& NOWAIT
+#endif
 #endif
 #ifdef ACCGPU
 #ifdef __HIP_PLATFORM_AMD__

diff --git a/src/trans/gpu/internal/trmtol_pack_unpack.F90 b/src/trans/gpu/internal/trmtol_pack_unpack.F90
@@ -136,12 +136,12 @@ SUBROUTINE TRMTOL_PACK(ALLOCATOR,HTRMTOL_PACK,ZOUTS,ZOUTA,ZOUTS0,ZOUTA0,FOUBUF_I
 #endif
 
 #ifdef OMPGPU
-    !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) PRIVATE(KM,ISL,IGLS,OFFSET1,OFFSET2,ZAOA,ZSOA) &
+    !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
+    !$OMP& PRIVATE(KM,ISL,IGLS,OFFSET1,OFFSET2,ZAOA,ZSOA) &
     !$OMP& FIRSTPRIVATE(KF_LEG,IOUT_STRIDES0,IOUT0_STRIDES0) &
+    !$OMP& SHARED(D,G,R,ZOUTS,ZOUTA,ZOUTS0,ZOUTA0,FOUBUF_IN) &
 #ifndef _CRAYFTN
-    !!TODO: Think about async for OMP
-    !!$OMP& NOWAIT
-    !$OMP&
+    !$OMP& NOWAIT
 #else
     !$OMP&
 #endif
@@ -283,13 +283,12 @@ SUBROUTINE TRMTOL_UNPACK(ALLOCATOR,HTRMTOL_UNPACK,FOUBUF,PREEL_COMPLEX,KF_CURREN
 OFFSET_VAR=D_NPTRLS(MYSETW)
 ILOEN_MAX=MAXVAL(G_NLOEN)
 #ifdef OMPGPU
-!$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) &
-!$OMP&       PRIVATE(IGLG,IOFF_LAT,ISTA,RET_REAL,RET_COMPLEX) &
-!$OMP&       FIRSTPRIVATE(KF_CURRENT,KF_TOTAL,OFFSET_VAR,ILOEN_MAX) &
+!$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
+!$OMP& PRIVATE(IGLG,IOFF_LAT,ISTA,RET_REAL,RET_COMPLEX) &
+!$OMP& FIRSTPRIVATE(KF_CURRENT,KF_TOTAL,OFFSET_VAR,ILOEN_MAX) &
+!$OMP& SHARED(D,G,FOUBUF,PREEL_COMPLEX) &
 #ifndef _CRAYFTN
-!!TODO: Think about async for OMP
-!!$OMP& NOWAIT
-!$OMP&
+!$OMP& NOWAIT
 #else
 !$OMP&
 #endif

diff --git a/src/trans/gpu/internal/vdtuv_mod.F90 b/src/trans/gpu/internal/vdtuv_mod.F90
@@ -106,12 +106,12 @@ SUBROUTINE VDTUV(KFIELD,PEPSNM,PVOR,PDIV,PU,PV)
 !              ------------------------------------------
 
 #ifdef OMPGPU
-!$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) PRIVATE(IR,II,KM,ZKM,JI) &
-!$OMP& FIRSTPRIVATE(KFIELD,KMLOC)
+!$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) DEFAULT(NONE) &
+!$OMP& PRIVATE(IR,II,KM,ZKM,JI) &
+!$OMP& FIRSTPRIVATE(KFIELD,KMLOC) &
+!$OMP& SHARED(D,R,F,PEPSNM,PVOR,PDIV,PU,PV) &
 #ifndef _CRAYFTN
-!!TODO: Think about async for OMP
-!!$OMP& NOWAIT
-!$OMP&
+!$OMP& NOWAIT
 #else
 !$OMP&
 #endif