Display

显示框架之app与SurfaceFlinger通信

Posted by Zhou Qi on 2023-07-04

显示框架之app与SurfaceFlinger通信

SurfaceFlinger是android显示的核心进程，在整个显示框架中起到一个承上启下的作用，“承上”指的是与app进程间的通信，“启下”指的是与Composer进程的通信。Surfaceflinger本身不进行绘制，是app数据上屏的中枢通路，先来看下SurfaceFlinger在整个显示流程中的位置。

显示流程图

从显示流程图看可知，SurfaceFlinger位于中间层的位置，目前的应用会调起renderthread使用GPU来渲染，应用侧使用surface来管理显示数据，surfaceflinger使用layer来对应应用侧的surface，surfaceflinger会根据合成的方式，选择device还是GPU合成，最后将layer数据提交给Composer进程，进而通过display 驱动上屏。本文就来研究下app和SurfaceFlinger之间的通信，主要针对native层的分析，代码基于android11。

APP与SurfaceFlinger之间的连接

首先框架会创建surfaceControl来管理surface，在jni层会创建一个surfaceComposerClient对象，这个对象是负责与SurfaceFlinger进程通信的重要载体。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
文件：frameworks/base/core/jni/android_view_SurfaceControl.cpp

static jlong nativeCreate(JNIEnv* env, jclass clazz, jobject sessionObj,
        jstring nameStr, jint w, jint h, jint format, jint flags, jlong parentObject,
        jobject metadataParcel) {
    ScopedUtfChars name(env, nameStr);
    sp<SurfaceComposerClient> client;
    if (sessionObj != NULL) {
        client = android_view_SurfaceSession_getClient(env, sessionObj);
    } else {
         // 如果java侧没有该对象则创建SurfaceComposerClient对象
        client = SurfaceComposerClient::getDefault();
    }
    SurfaceControl *parent = reinterpret_cast<SurfaceControl*>(parentObject);
    sp<SurfaceControl> surface;
   ...
    // 通过SurfaceComposerClient创建一个Surface
    status_t err = client->createSurfaceChecked(
            String8(name.c_str()), w, h, format, &surface, flags, parent, std::move(metadata));
   ...
    // 返回给上层surface对象
    return reinterpret_cast<jlong>(surface.get());
}

文件：frameworks/native/libs/gui/SurfaceComposerClient.cpp 

sp<SurfaceComposerClient> SurfaceComposerClient::getDefault() {
    return DefaultComposerClient::getComposerClient();
 }

class DefaultComposerClient: public Singleton<DefaultComposerClient> {
    Mutex mLock;
    sp<SurfaceComposerClient> mClient;
    friend class Singleton<ComposerService>;
public:
    static sp<SurfaceComposerClient> getComposerClient() {
        // 使用单例模式创建DefaultComposerClient 对象
        DefaultComposerClient& dc = DefaultComposerClient::getInstance();
        Mutex::Autolock _l(dc.mLock);
        if (dc.mClient == nullptr) {
            // 创建SurfaceComposerClient 对象
            dc.mClient = new SurfaceComposerClient;
        }
        return dc.mClient;
    }
};
// 由于SurfaceComposerClient是sp指针，第一次创建时会执行onFirstRef 函数
void SurfaceComposerClient::onFirstRef() {
    sp<ISurfaceComposer> sf(ComposerService::getComposerService());
    if (sf != nullptr && mStatus == NO_INIT) {
        sp<ISurfaceComposerClient> conn;
        conn = sf->createConnection();
        if (conn != nullptr) {
            mClient = conn;
            mStatus = NO_ERROR;
        }
    }
}
//在getComposerService 函数中调connectLocked来get到SurfaceFlinger服务并注册了死亡监听
void ComposerService::connectLocked() {
    const String16 name("SurfaceFlinger");
    while (getService(name, &mComposerService) != NO_ERROR) {
        usleep(250000);
    }
    assert(mComposerService != nullptr);

    // Create the death listener.
    class DeathObserver : public IBinder::DeathRecipient {
        ComposerService& mComposerService;
        virtual void binderDied(const wp<IBinder>& who) {
            ALOGW("ComposerService remote (surfaceflinger) died [%p]",
                  who.unsafe_get());
            mComposerService.composerServiceDied();
        }
     public:
        explicit DeathObserver(ComposerService& mgr) : mComposerService(mgr) { }
    };

    mDeathObserver = new DeathObserver(*const_cast<ComposerService*>(this));
    IInterface::asBinder(mComposerService)->linkToDeath(mDeathObserver);
}
文件：frameworks/native/services/surfaceflinger/surfaceflinger.cpp

// 与SurfaceFlinger建立联系，Client持有SurfaceFlinger对象
sp<ISurfaceComposerClient> SurfaceFlinger::createConnection() {
    const sp<Client> client = new Client(this);
    return client->initCheck() == NO_ERROR ? client : nullptr;
}

看注释，至此完成了从SurfaceControl-> SurfaceComposerClient -> SurfaceFlinger的连接过程。应用要创建Surface时，对应SurfaceFlinger会创建layer与之对应。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
文件：frameworks/native/libs/gui/SurfaceComposerClient.cpp 

status_t SurfaceComposerClient::createSurfaceChecked(const String8& name, uint32_t w, uint32_t h,
                                                     PixelFormat format,
                                                     sp<SurfaceControl>* outSurface, uint32_t flags,
                                                     SurfaceControl* parent, LayerMetadata metadata,
                                                     uint32_t* outTransformHint) {
    sp<SurfaceControl> sur;
    ...
        // 会执行到Client.cpp里面的createSurface
        err = mClient->createSurface(name, w, h, format, flags, parentHandle, std::move(metadata),
                                     &handle, &gbp, &transformHint);
       ...
        ALOGE_IF(err, "SurfaceComposerClient::createSurface error %s", strerror(-err));
        if (err == NO_ERROR) {
            // 根据返回的handle 和 gbp 创建 SurfaceControl
            *outSurface = new SurfaceControl(this, handle, gbp, transformHint);
        }
    }
    return err;
}

文件: frameworks/native/services/surfaceflinger/Client.cpp

status_t Client::createSurface(const String8& name, uint32_t w, uint32_t h, PixelFormat format,
                               uint32_t flags, const sp<IBinder>& parentHandle,
                               LayerMetadata metadata, sp<IBinder>* handle,
                               sp<IGraphicBufferProducer>* gbp, uint32_t* outTransformHint) {
    // We rely on createLayer to check permissions.
    return mFlinger->createLayer(name, this, w, h, format, flags, std::move(metadata), handle, gbp,
                                 parentHandle, nullptr, outTransformHint);
}

文件： frameworks/native/services/surfaceflinger/SurfaceFlinger.cpp

tatus_t SurfaceFlinger::createLayer(const String8& name, const sp<Client>& client, uint32_t w,
                                     uint32_t h, PixelFormat format, uint32_t flags,
                                     LayerMetadata metadata, sp<IBinder>* handle,
                                     sp<IGraphicBufferProducer>* gbp,
                                     const sp<IBinder>& parentHandle, const sp<Layer>& parentLayer,
   ...
    sp<Layer> layer;
   ...
    switch (flags & ISurfaceComposerClient::eFXSurfaceMask) {
        case ISurfaceComposerClient::eFXSurfaceBufferQueue:
            // 对于surface类型创建layer类型，主要看下带有BufferQueue的layer
            result = createBufferQueueLayer(client, std::move(uniqueName), w, h, flags,
                                            std::move(metadata), format, handle, gbp, &layer);

            break;
    ...
    // 将创建的layer放到mCurrentState 里面，SurfaceFlinger内部管理了mCurrentState 和 mDrawingState
    result = addClientLayer(client, *handle, *gbp, layer, parentHandle, parentLayer,
                            addToCurrentState, outTransformHint);
    ...
    setTransactionFlags(eTransactionNeeded);
    return result;
}
文件： frameworks/native/services/surfaceflinger/SurfaceFlinger.cpp

status_t SurfaceFlinger::createBufferQueueLayer(const sp<Client>& client, std::string name,
                                                uint32_t w, uint32_t h, uint32_t flags,
                                                LayerMetadata metadata, PixelFormat& format,
                                                sp<IBinder>* handle,
                                                sp<IGraphicBufferProducer>* gbp,
                                                sp<Layer>* outLayer) {
    ...
    sp<BufferQueueLayer> layer;
    LayerCreationArgs args(this, client, std::move(name), w, h, flags, std::move(metadata));
    ...
        Mutex::Autolock lock(mStateLock);
        layer = getFactory().createBufferQueueLayer(args);
    ...
    if (err == NO_ERROR) {
         // 在surfaceflinger侧 new了一个handle，这个handle指向这个layer
        *handle = layer->getHandle();
        //gbp是创建的BufferQueueProducer对象
        *gbp = layer->getProducer();
        *outLayer = layer;
    }
        
    ALOGE_IF(err, "createBufferQueueLayer() failed (%s)", strerror(-err));
    return err;
}

文件： frameworks/native/services/surfaceflinger/SurfaceFlingerDefaultFactory.cpp

sp<BufferQueueLayer> DefaultFactory::createBufferQueueLayer(const LayerCreationArgs& args) {
    // new 了BufferQueueLayer对象，sp指针第一次创建时会执行onFirstRef 函数
    return new BufferQueueLayer(args);
}

文件：frameworks/native/services/surfaceflinger/BufferQueueLayer.cpp

void BufferQueueLayer::onFirstRef() {
    BufferLayer::onFirstRef();

    // Creates a custom BufferQueue for SurfaceFlingerConsumer to use
    sp<IGraphicBufferProducer> producer;
    sp<IGraphicBufferConsumer> consumer;
    // 创建一个BufferQueue对象，跟着创建一个BufferQueueCore, BufferQueueProducer和 BufferQueueConsumer对象
    mFlinger->getFactory().createBufferQueue(&producer, &consumer, true);
    // 对producer 的一个封装，MonitoredProducer实质与 producer无差别
    mProducer = mFlinger->getFactory().createMonitoredProducer(producer, mFlinger, this);
    // 创建BufferLayerConsumer ，首先创建父类ConsumerBase 对象
    mConsumer =
            mFlinger->getFactory().createBufferLayerConsumer(consumer, mFlinger->getRenderEngine(),
                                                             mTextureName, this);
   ...
    // 注册了一个监听，将BufferQueueLayer的函数方法注册到BufferQueueCore里面，说明当Buffer有变化时会通知BufferQueueLayer执行相应的动作
    mContentsChangedListener = new ContentsChangedListener(this);
    mConsumer->setContentsChangedListener(mContentsChangedListener);
    mConsumer->setName(String8(mName.data(), mName.size()));

    // BufferQueueCore::mMaxDequeuedBufferCount is default to 1
    // 设置producer最大dequeue的buffer数量
    if (!mFlinger->isLayerTripleBufferingDisabled()) {
        mProducer->setMaxDequeuedBufferCount(2);
    }
}

文件：frameworks/native/services/surfaceflinger/Layer.cpp

sp<IBinder> Layer::getHandle() {
    Mutex::Autolock _l(mLock);
    if (mGetHandleCalled) {
        ALOGE("Get handle called twice" );
        return nullptr;
    }
    mGetHandleCalled = true;
   // 创建一个handle对应layer
    return new Handle(mFlinger, this);
}

看注释，SurfaceFlinger侧创建了handle和GraphicsBufferProducer对象给到SurfaceControl，handle 指向surfaceflinger的layer，GraphicsBufferProducer 提供surface对Buffer的操作接口，上层操作surface的handle来作用到surfaceflinger的layer。至此，surfaceflinger创建layer的流程分析完了，比较绕的地方是涉及BufferQueue对象之间的关系，用一副类的关系图来说明之间的关系。

BufferQueue类图

APP与SurfaceFlinger之间数据的传递

现在应用一般采用GPU去渲染加速，GPU操作在RenderThread里(HWUI 里面的逻辑很复杂，后续会进行剖析)，应用要绘制数据首先需要申请一块Buffer，由前面分析可知，应用是一个GraphicsBufferProducer的角色，故可以通过dequeueBuffer来向surfaceflinger申请一块Buffer。

DequeueBuffer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
文件： frameworks/native/libs/gui/Surface.cpp

int Surface::dequeueBuffer(android_native_buffer_t** buffer, int* fenceFd) {
    ATRACE_CALL();
    ALOGV("Surface::dequeueBuffer");

   ...
    // 从应用进程调到SurfaceFlinger进程，实现在SurfaceFlinger侧
    status_t result = mGraphicBufferProducer->dequeueBuffer(&buf, &fence, reqWidth, reqHeight,
                                                            reqFormat, reqUsage, &mBufferAge,
                                                            enableFrameTimestamps ? &frameTimestamps
                                                                                  : nullptr);
    ...
     // 后续会通过requestBuffer拿到Buffer
    sp<GraphicBuffer>& gbuf(mSlots[buf].buffer);

    ...
        // requestBuffer将通过importBuffer将Buffer映射到应用进程
        result = mGraphicBufferProducer->requestBuffer(buf, &gbuf);
    ...
       *buffer = gbuf.get();
    ...

       mDequeuedSlots.insert(buf);
       return OK;
}

文件： frameworks/native/libs/gui/BufferQueueProducer.cpp
status_t BufferQueueProducer::dequeueBuffer(int* outSlot, sp<android::Fence>* outFence,
                                            uint32_t width, uint32_t height, PixelFormat format,
                                            uint64_t usage, uint64_t* outBufferAge,
                                            FrameEventHistoryDelta* outTimestamps) {
    ...
    int found = BufferItem::INVALID_BUFFER_SLOT;
    while (found == BufferItem::INVALID_BUFFER_SLOT) {
          // 优先从mFreeBuffers 里面获取slot, 若没有则从mFreeSlots 里面获取，从mFreeSlots 里面获取的都是还未分配Buffer的slot
          status_t status = waitForFreeSlotThenRelock(FreeSlotCaller::Dequeue, lock, &found);
    ...
    }
    if (returnFlags & BUFFER_NEEDS_REALLOCATION) {
        ...
        //若得到的slot对应的buffer为空则重新分配一块GraphicsBuffer
        sp<GraphicBuffer> graphicBuffer = new GraphicBuffer(
                width, height, format, BQ_LAYER_COUNT, usage,
                {mConsumerName.string(), mConsumerName.size()});
        ...
        mSlots[*outSlot].mGraphicBuffer = graphicBuffer;
        ...
}

文件： frameworks/native/libs/gui/BufferQueueProducer.cpp

status_t BufferQueueProducer::requestBuffer(int slot, sp<GraphicBuffer>* buf) {
    ...
    mSlots[slot].mRequestBufferCalled = true;
     // GraphicsBuffer跨进程传递执行unflatten
    *buf = mSlots[slot].mGraphicBuffer;
    return NO_ERROR;
}

文件： frameworks/native/libs/ui/GraphicBuffer.cpp

status_t GraphicBuffer::unflatten(void const*& buffer, size_t& size, int const*& fds,
                                  size_t& count) {
...

if (handle != nullptr) {
        buffer_handle_t importedHandle;
        status_t err = mBufferMapper.importBuffer(handle, uint32_t(width), uint32_t(height),
                uint32_t(layerCount), format, usage, uint32_t(stride), &importedHandle);
...
        handle = importedHandle;
...

       return NO_ERROR;
}

应用的surface执行dequeueBuffer是在应用进程，具体实现是通过BufferQueueProducer的dequeueBuffer，是在SurfaceFlinger进程，所以需要通过requestBuffer将Buffer映射到应用进程，具体实现是通过importBuffer。真正分配Buffer的地方是在ion，ion是怎么分配Buffer的，以及Buffer怎么实现app和SurfaceFlinger共享，先用一张简图概述下，后续会分析。

Buffer共享

至此app就拿到了可绘制的Buffer，GPU可以将数据渲染到这块Buffer上。GPU将数据渲染到Buffer后，会执行eglSwapBuffers交换front和back buffer，eglSwapBuffers会调用queueBuffer，android系统用queueBuffer来请求vsync 唤醒SurfaceFlinger刷新。

queueBuffer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
文件: frameworks/native/libs/gui/Surface.cpp

int Surface::queueBuffer(android_native_buffer_t* buffer, int fenceFd) {
    ...
     // dequeue出来的buffer对应的slot
    int i = getSlotFromBufferLocked(buffer);
    ...
    // 传入一些参数，其中 fenceFd 是由GPU带过来的，表示GPU渲染是否完成，这里为acquireFence
    IGraphicBufferProducer::QueueBufferInput input(timestamp, isAutoTimestamp,
            static_cast<android_dataspace>(mDataSpace), crop, mScalingMode,
            mTransform ^ mStickyTransform, fence, mStickyTransform,
            mEnableFrameTimestamps);
    ...
    // 调到SurfaceFlinger进程的queueBuffer
    status_t err = mGraphicBufferProducer->queueBuffer(i, input, &output);
    ...
}

文件： frameworks/native/libs/gui/BufferQueueProducer.cpp

status_t BufferQueueProducer::queueBuffer(int slot,
        const QueueBufferInput &input, QueueBufferOutput *output) {
...
    BufferItem item;
...
        // 将数据同步到item对象
        item.mAcquireCalled = mSlots[slot].mAcquireCalled;
        item.mGraphicBuffer = mSlots[slot].mGraphicBuffer;
        item.mCrop = crop;
        item.mTransform = transform &
                ~static_cast<uint32_t>(NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY);
        item.mTransformToDisplayInverse =
                (transform & NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY) != 0;
        item.mScalingMode = static_cast<uint32_t>(scalingMode);
        item.mTimestamp = requestedPresentTimestamp;
        item.mIsAutoTimestamp = isAutoTimestamp;
        item.mDataSpace = dataSpace;
        item.mHdrMetadata = hdrMetadata;
        item.mFrameNumber = currentFrameNumber;
        item.mSlot = slot;
        item.mFence = acquireFence;
        item.mFenceTime = acquireFenceTime;
        item.mIsDroppable = mCore->mAsyncMode ||
                (mConsumerIsSurfaceFlinger && mCore->mQueueBufferCanDrop) ||
                (mCore->mLegacyBufferDrop && mCore->mQueueBufferCanDrop) ||
                (mCore->mSharedBufferMode && mCore->mSharedBufferSlot == slot);
        item.mSurfaceDamage = surfaceDamage;
        item.mQueuedBuffer = true;
        item.mAutoRefresh = mCore->mSharedBufferMode && mCore->mAutoRefresh;
        item.mApi = mCore->mConnectedApi;
       ...
      
       // 把item放到mQueue里面
       mCore->mQueue.push_back(item);
       // 这里的mCore->mConsumerListener 实际上为ConsumerBase对象
       frameAvailableListener = mCore->mConsumerListener;
       ...
      if (frameAvailableListener != nullptr) {
        frameAvailableListener->onFrameAvailable(item);
      }...
}

文件： frameworks/native/libs/gui/ConsumerBase.cpp

void ConsumerBase::onFrameAvailable(const BufferItem& item) {
    CB_LOGV("onFrameAvailable");

    sp<FrameAvailableListener> listener;
    { // scope for the lock
        Mutex::Autolock lock(mFrameAvailableMutex);
        listener = mFrameAvailableListener.promote();
    }

    if (listener != nullptr) {
        CB_LOGV("actually calling onFrameAvailable");
        // listener实际上是BufferQueueLayer对象
        listener->onFrameAvailable(item);
    }
}

根据类图关系，frameAvailableListener 实际上是ConsumerBase 对象，在ConsumerBase 初始化的时候进行赋值，通过 mConsumer->consumerConnect(proxy, controlledByApp)接口，所以会调到ConsumerBase::onFrameAvailable， 而mFrameAvailableListener 是BufferQueueLayer -> setContentsChangedListener(mContentsChangedListener) 设的，实际上是 BufferQueueLayer对象，因此最后走到BufferQueueLayer::onFrameAvailable 逻辑里面了。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
文件： frameworks/native/services/surfaceflinger/BufferQueueLayer.cpp

void BufferQueueLayer::onFrameAvailable(const BufferItem& item) {
   ...
    // 将item 放到mQueueItems里面
    mQueueItems.push_back(item);
    mQueuedFrames++;
   ...
    // 申请vsync去触发surfaceflinger刷新
    mFlinger->signalLayerUpdate();
    mConsumer->onBufferAvailable(item);
}

文件： frameworks/native/services/surfaceflinger/Scheduler/EventThread.cpp

void EventThread::requestNextVsync(const sp<EventThreadConnection>& connection) {
    if (connection->resyncCallback) {
        connection->resyncCallback();
    }
    std::lock_guard<std::mutex> lock(mMutex);

    if (connection->vsyncRequest == VSyncRequest::None) {
        // Vsync请求置为Single
        connection->vsyncRequest = VSyncRequest::Single;
        mCondition.notify_all();
    }
}

让Surfaceflinger刷新需要2个必要条件： 1. 有requestNextVsync 的请求 2. 软件vsync计算模型回调 onVSyncEvent创建一个vsyncEvent，涉及到vsync模型，后面会介绍。
到这里，app完成了与SurfaceFlinger的连接以及将应用数据传给到SurfaceFlinger。这里需要注意的是，其实SurfaceFlinger并不会动Buffer里面的数据，只是中间传递的一个过程，Buffer数据是由GPU渲染，display显示，下篇分析下SurfaceFlinger在刷新时做了哪些动作。

This is copyright.

FEATURED TAGS
Display
FRIENDS