Changeset 146:e94468699f12

Timestamp:

03/13/2012 10:13:58 PM (15 months ago)

Author:

Andreas Schaefer <gentryx@…>

Branch:

default

Message:

changed HiParSimulator?'s event handling so that ParallelWriters? can be correctly notified at simulation start of both, inner ghost zones and inner set regions

Location:

src/parallelization

Files:

• hiparsimulator.h (modified) (3 diffs)
• hiparsimulator/parallelwriteradapter.h (modified) (3 diffs)
• hiparsimulator/updategroup.h (modified) (2 diffs)
• hiparsimulator/vanillastepper.h (modified) (7 diffs)

src/parallelization/hiparsimulator.h

@@ -48 +48 @@
     typedef UpdateGroup<CELL_TYPE, PARTITION> UpdateGroupType;
     typedef typename ParentType::GridType GridType;
+    typedef ParallelWriterAdapter<typename UpdateGroupType::GridType, CELL_TYPE, PARTITION> ParallelWriterAdapterType;
     static const int DIM = Topology::DIMENSIONS;
 
@@ -62 +63 @@
         communicator(_communicator)
     {
+    }   
+
+    // fixme: need test
+    inline void run()
+    {
+        initUpdateGroup();
+
+        // fixme: use events here
+        std::pair<int, int> currentStep = updateGroup->currentStep();
+        unsigned remainingSteps = 
+            this->initializer->maxSteps() - currentStep.first;
+        unsigned remainingNanoSteps = 
+            remainingSteps * CELL_TYPE::nanoSteps() - currentStep.second;
+        nanoStep(remainingNanoSteps);
+    }
+
+    // fixme: need test
+    inline void step()
+    {
+        initUpdateGroup();
+
+        nanoStep(CELL_TYPE::nanoSteps());
+    }
+
+    // fixme: need test
+    virtual void getGridFragment(
+        const GridType **grid, 
+        const Region<DIM> **validRegion) 
+    {
+        *grid = &updateGroup->grid();
+        // fixme: we can't even guarantee this part
+        *validRegion = &partitionManager.ownRegion();
+    }
+
+    virtual unsigned getStep() const 
+    {
+        if (updateGroup) {
+            return updateGroup->currentStep().first;
+        } else {
+            return this->initializer->startStep();
+        }
+    }
+
+    virtual void registerWriter(ParallelWriter<CELL_TYPE> *writer)
+    {
+        DistributedSimulator<CELL_TYPE>::registerWriter(writer);
+
+        // we need two adapters as each ParallelWriter needs to be
+        // notified twice: once for the (inner) ghost zone, and once
+        // for the inner set.
+        typename UpdateGroupType::PatchAccepterPtr adapterGhost(
+            new ParallelWriterAdapterType(
+                this,
+                this->getWriters().back(),
+                this->initializer->startStep(),
+                this->initializer->maxSteps()));
+        typename UpdateGroupType::PatchAccepterPtr adapterInnerSet(
+            new ParallelWriterAdapterType(
+                this, 
+                this->getWriters().back(),
+                this->initializer->startStep(),
+                this->initializer->maxSteps()));
+
+        writerAdaptersGhost.push_back(adapterGhost);
+        writerAdaptersInner.push_back(adapterInnerSet);
+    }
+
+private:
+    boost::shared_ptr<LoadBalancer> balancer;
+    unsigned loadBalancingPeriod;
+    unsigned ghostZoneWidth;
+    EventMap events;
+    PartitionManager<DIM, Topology> partitionManager;
+    MPI::Comm *communicator;
+    boost::shared_ptr<UpdateGroupType> updateGroup;
+    typename UpdateGroupType::PatchAccepterVec writerAdaptersGhost;
+    typename UpdateGroupType::PatchAccepterVec writerAdaptersInner;
+
+    SuperVector<long> initialWeights(const long& items, const long& size) const
+    {    
+        SuperVector<long> ret(size);
+        long lastPos = 0;
+
+        for (long i = 0; i < size; i++) {
+            long currentPos = items * (i + 1) / size;
+            ret[i] = currentPos - lastPos;
+            lastPos = currentPos;
+        }
+
+        return ret;
+    }
+
+    inline void nanoStep(const unsigned& s)
+    {
+        std::cout << "nanoStep(" << s << ")\n";
+        updateGroup->update(s);
+
+        // fixme: honor events here:
+        // unsigned endNanoStep = nanoStepCounter + s;
+        // events[endNanoStep].insert(PAUSE);
+        
+        // while (nanoStepCounter < endNanoStep) {
+        //     std::pair<unsigned, EventSet> currentEvents = extractCurrentEvents();
+        //     nanoStepCounter = currentEvents.first;
+        //     handleEvents(currentEvents.second);
+        // }
+    }
+
+    /**
+     * We need to do late/lazy initialization to give the user time to
+     * add ParallelWriter objects before calling run(). Writers may
+     * only be added savely to an UpdateGroup upon creation because of
+     * the way the Stepper handles ghostzone updates. It's a long
+     * story... At the end of the day this remains the best compromise
+     * of hiding complexity (in the Stepper) and a convenient API of
+     * the Simulator on the one hand, and avoiding objects with an
+     * uninitialized state on the other.
+     */
+    inline void initUpdateGroup()
+    {
+        if (updateGroup) {
+            return;
+        }
+
         CoordBox<DIM> box = this->initializer->gridBox();
         updateGroup.reset(
@@ -71 +196 @@
                 ghostZoneWidth,
                 this->initializer,
+                writerAdaptersGhost,
+                writerAdaptersInner,
                 communicator));
-    }   
-
-    // fixme: need test
-    inline void run()
-    {
-        // fixme: use events here
-        std::pair<int, int> currentStep = updateGroup->currentStep();
-        unsigned remainingSteps = 
-            this->initializer->maxSteps() - currentStep.first;
-        unsigned remainingNanoSteps = 
-            remainingSteps * CELL_TYPE::nanoSteps() - currentStep.second;
-        nanoStep(remainingNanoSteps);
-    }
-
-    // fixme: need test
-    inline void step()
-    {
-        nanoStep(CELL_TYPE::nanoSteps());
-    }
-
-    // fixme: need test
-    virtual void getGridFragment(
-        const GridType **grid, 
-        const Region<DIM> **validRegion) 
-    {
-        *grid = &updateGroup->grid();
-        // fixme: we can't even guarantee this part
-        *validRegion = &partitionManager.ownRegion();
-    }
-
-    virtual unsigned getStep() const 
-    {
-        return updateGroup->currentStep().first;
-    }
-
-    virtual void registerWriter(ParallelWriter<CELL_TYPE> *writer)
-    {
-        DistributedSimulator<CELL_TYPE>::registerWriter(writer);
-
-        typename UpdateGroupType::PatchAccepterPtr adapter(
-            new ParallelWriterAdapter<
-                typename UpdateGroupType::GridType, 
-                CELL_TYPE, 
-                PARTITION>(
-                this, 
-                this->getWriters().back()));
-        // fixme: use different adapters here to avoid nanostep confusion b/c of ghostzone updates
-        // updateGroup->addPatchAccepter(adapter, Stepper<CELL_TYPE>::GHOST);
-        updateGroup->addPatchAccepter(adapter, Stepper<CELL_TYPE>::INNER_SET);
-    }
-
-private:
-    boost::shared_ptr<LoadBalancer> balancer;
-    unsigned loadBalancingPeriod;
-    unsigned ghostZoneWidth;
-    // fixme: need this?
-    EventMap events;
-    PartitionManager<DIM, Topology> partitionManager;
-    MPI::Comm *communicator;
-    boost::shared_ptr<UpdateGroupType> updateGroup;
-
-    SuperVector<long> initialWeights(const long& items, const long& size) const
-    {    
-        SuperVector<long> ret(size);
-        long lastPos = 0;
-
-        for (long i = 0; i < size; i++) {
-            long currentPos = items * (i + 1) / size;
-            ret[i] = currentPos - lastPos;
-            lastPos = currentPos;
-        }
-
-        return ret;
-    }
-
-    inline void nanoStep(const unsigned& s)
-    {
-        std::cout << "nanoStep(" << s << ")\n";
-        updateGroup->update(s);
-
-        // fixme: honor events here:
-        // unsigned endNanoStep = nanoStepCounter + s;
-        // events[endNanoStep].insert(PAUSE);
-        
-        // while (nanoStepCounter < endNanoStep) {
-        //     std::pair<unsigned, EventSet> currentEvents = extractCurrentEvents();
-        //     nanoStepCounter = currentEvents.first;
-        //     handleEvents(currentEvents.second);
-        // }
+
+        writerAdaptersGhost.clear();
+        writerAdaptersInner.clear();
     }
 

src/parallelization/hiparsimulator/parallelwriteradapter.h

@@ -24 +24 @@
     ParallelWriterAdapter(
         HiParSimulatorType *_sim,
-        boost::shared_ptr<ParallelWriter<CELL_TYPE> > _writer) :
+        boost::shared_ptr<ParallelWriter<CELL_TYPE> > _writer,
+        const long& firstStep,
+        const long& lastStep) :
         sim(_sim),
-        writer(_writer)
+        writer(_writer),
+        firstNanoStep(firstStep * CELL_TYPE::nanoSteps()),
+        lastNanoStep(lastStep   * CELL_TYPE::nanoSteps())
     {
-        reload();
+        reload(firstNanoStep);
+        reload(lastNanoStep);
     }
 
@@ -36 +41 @@
         const long& nanoStep) 
     {
-        std::cout << "bingobongoA " << nanoStep << "\n";
         if (!this->checkNanoStepPut(nanoStep))
             return;
         this->requestedNanoSteps.erase_min();
 
-        std::cout << "bingobongoB " << nanoStep << "\n";
-        // fixme: load next event
         // fixme: set simulator up to link to correct grid/validRegion
-        // writer->stepFinished();
+        if (nanoStep == firstNanoStep) {
+            writer->initialized();
+        } else {
+            if (nanoStep == lastNanoStep) {
+                writer->allDone();
+            } else {
+                writer->stepFinished();
+            }
+        }
+
         reload();
     }
@@ -51 +62 @@
     HiParSimulatorType *sim;
     boost::shared_ptr<ParallelWriter<CELL_TYPE> > writer;
+    long firstNanoStep;
+    long lastNanoStep;
 
-    long nextOutputStep()
+    long nextOutputStep(const long& step)
     {
-        long step = sim->getStep();
         long remainder = step % writer->getPeriod();
-        step += writer->getPeriod() - remainder;
-        return step;
+        long next = step + writer->getPeriod() - remainder;
+        return next;
     }
 
     void reload()
     {
-        long nextNanoStep = nextOutputStep() * CELL_TYPE::nanoSteps();
-        std::cout << "nextNanoStep: " << nextNanoStep << "\n";
+        long nextNanoStep = nextOutputStep(sim->getStep()) * CELL_TYPE::nanoSteps();
+        reload(nextNanoStep);
+    }
+
+    void reload(const long& nextNanoStep)
+    {
         this->pushRequest(nextNanoStep);
     }

src/parallelization/hiparsimulator/updategroup.h

@@ -45 +45 @@
         const unsigned& _ghostZoneWidth,
         Initializer<CELL_TYPE> *_initializer,
+        PatchAccepterVec patchAcceptersGhost=PatchAccepterVec(),
+        PatchAccepterVec patchAcceptersInner=PatchAccepterVec(),
         MPI::Comm *communicator = &MPI::COMM_WORLD) : 
         partition(_partition),
@@ -90 +92 @@
         }
 
-        stepper.reset(new STEPPER(partitionManager, initializer, ghostZoneAccepterLinks));
+        stepper.reset(new STEPPER(
+                          partitionManager, 
+                          initializer,
+                          patchAcceptersGhost + ghostZoneAccepterLinks,
+                          patchAcceptersInner));
 
         // the ghostzone receivers may be safely added after

src/parallelization/hiparsimulator/vanillastepper.h

@@ -29 +29 @@
         boost::shared_ptr<MyPartitionManager> _partitionManager,
         Initializer<CELL_TYPE> *_initializer,
-        const PatchAccepterVec ghostZonePatchAccepters = PatchAccepterVec()) :
+        const PatchAccepterVec ghostZonePatchAccepters = PatchAccepterVec(),
+        const PatchAccepterVec innerSetPatchAccepters = PatchAccepterVec()) :
         ParentType(_partitionManager, _initializer)
     {
@@ -37 +38 @@
         for (int i = 0; i < ghostZonePatchAccepters.size(); ++i) {
             addPatchAccepter(ghostZonePatchAccepters[i], ParentType::GHOST);
+        }
+        for (int i = 0; i < innerSetPatchAccepters.size(); ++i) {
+            addPatchAccepter(innerSetPatchAccepters[i], ParentType::INNER_SET);
         }
 
@@ -101 +105 @@
         const long& nanoStep)
     {
-        std::cout << "notifyPatchAccepters(" << nanoStep << ")\n";
         for (class ParentType::PatchAccepterList::iterator i = 
                  this->patchAccepters[patchType].begin();
              i != this->patchAccepters[patchType].end();
              ++i) {
-            std::cout << "  @ " << (*i)->nextRequiredNanoStep() << "\n";
             if (nanoStep == (*i)->nextRequiredNanoStep()) {
                 (*i)->put(*oldGrid, region, nanoStep);
@@ -121 +123 @@
                  this->patchProviders[patchType].begin();
              i != this->patchProviders[patchType].end();
-             ++i)
+             ++i) {
             (*i)->get(
                 &*oldGrid,
                 region,
                 nanoStep);
+        }
     }
 
@@ -144 +147 @@
         newGrid->getEdgeCell() = oldGrid->getEdgeCell();
         resetValidGhostZoneWidth();
+
+        notifyPatchAccepters(
+            rim(),     
+            ParentType::GHOST,     
+            globalNanoStep());
+        notifyPatchAccepters(
+            getPartitionManager().innerSet(0), 
+            ParentType::INNER_SET, 
+            globalNanoStep());
 
         kernelBuffer = MyPatchBuffer1(getPartitionManager().getVolatileKernel());
@@ -176 +188 @@
         int oldNanoStep = curNanoStep;
         int oldStep = curStep;
+        int curGlobalNanoStep = globalNanoStep();
+
         for (int t = 0; t < ghostZoneWidth(); ++t) {
             const Region<DIM>& region = getPartitionManager().rim(t + 1);
@@ -191 +205 @@
 
             std::swap(oldGrid, newGrid);
+
+            ++curGlobalNanoStep;
+            notifyPatchAccepters(rim(), ParentType::GHOST, curGlobalNanoStep);
         }
         curNanoStep = oldNanoStep;
         curStep = oldStep;
 
-        // offset ghostZoneWidth() for nanoStep needed as ghost zone
-        // updates preceede updates to the inner set.
-        int nextNanoStep = globalNanoStep() + ghostZoneWidth();
-        notifyPatchAccepters(rim(), ParentType::GHOST, nextNanoStep);
-                   saveRim(nextNanoStep);
+        saveRim(curGlobalNanoStep);
         if (ghostZoneWidth() % 2)
             std::swap(oldGrid, newGrid);